From c2203d19c11af00fa0be69c5500c2c1b9e1bb431 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 16 May 2017 18:54:44 -0500
Subject: [PATCH 001/100] wip on adding RLE to roaring, add, countRange,
 contains

---
 roaring/roaring.go               | 148 ++++++++++++++++++++++++++++---
 roaring/roaring_internal_test.go | 139 ++++++++++++++++++++++++++++-
 2 files changed, 275 insertions(+), 12 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index c5a8b1650..743513562 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -902,7 +902,23 @@ type container struct {
 	n      int      // number of integers in container
 	array  []uint32 // used for array containers
 	bitmap []uint64 // used for bitmap containers
-	mapped bool     // mapped directly to a byte slice when true
+	runs   []interval16
+	mapped bool // mapped directly to a byte slice when true
+}
+
+type interval16 struct {
+	start uint16
+	last  uint16
+}
+
+// runlen returns the count of integers in the interval.
+func (iv interval16) runlen() int {
+	return 1 + int(iv.last) - int(iv.start)
+}
+
+// String produces a human viewable string of the contents.
+func (iv interval16) String() string {
+	return fmt.Sprintf("[%d, %d]", iv.start, iv.last)
 }
 
 // newContainer returns a new instance of container.
@@ -911,7 +927,17 @@ func newContainer() *container {
 }
 
 // isArray returns true if the container is an array container.
-func (c *container) isArray() bool { return c.bitmap == nil }
+func (c *container) isArray() bool {
+	return c.bitmap == nil && c.runs == nil
+}
+
+func (c *container) isBitmap() bool {
+	return c.array == nil && c.runs == nil
+}
+
+func (c *container) isRun() bool {
+	return c.array == nil && c.bitmap == nil
+}
 
 // unmap creates copies of the containers data in the heap.
 //
@@ -932,6 +958,11 @@ func (c *container) unmap() {
 		copy(tmp, c.bitmap)
 		c.bitmap = tmp
 	}
+	if c.runs != nil {
+		tmp := make([]interval16, len(c.runs))
+		copy(tmp, c.runs)
+		c.runs = tmp
+	}
 	c.mapped = false
 }
 
@@ -944,8 +975,10 @@ func (c *container) count() (n int) {
 func (c *container) countRange(start, end uint32) (n int) {
 	if c.isArray() {
 		return c.arrayCountRange(start, end)
+	} else if c.isBitmap() {
+		return c.bitmapCountRange(start, end)
 	}
-	return c.bitmapCountRange(start, end)
+	return c.runCountRange(start, end)
 }
 
 func (c *container) arrayCountRange(start, end uint32) (n int) {
@@ -991,12 +1024,50 @@ func (c *container) bitmapCountRange(start, end uint32) int {
 	return int(n)
 }
 
-// add adds a value to the container.
-func (c *container) add(v uint32) bool {
-	if c.isArray() {
-		return c.arrayAdd(v)
+func (c *container) runCountRange(start, end uint32) (n int) {
+	start16, end16 := uint16(start), uint16(end)
+	for _, iv := range c.runs {
+		// iv is before range
+		if iv.last < start16 {
+			continue
+		}
+		// iv is after range
+		if end16 < iv.start {
+			break
+		}
+		// iv is superset of range
+		if iv.start < start16 && iv.last > end16 {
+			return int(end16 - start16)
+		}
+		// iv is subset of range
+		if iv.start >= start16 && iv.last < end16 {
+			n += iv.runlen()
+		}
+		// iv overlaps beginning of range
+		if iv.start < start16 && iv.last < end16 {
+			n += int(iv.last - start16 + 1)
+		}
+		// iv overlaps end of range
+		if iv.start > start16 && iv.last >= end16 {
+			n += int(end16 - iv.start)
+		}
 	}
-	return c.bitmapAdd(v)
+	return n
+}
+
+// add adds a value to the container.
+func (c *container) add(v uint32) (added bool) {
+	if c.isArray() {
+		added = c.arrayAdd(v)
+	} else if c.isBitmap() {
+		added = c.bitmapAdd(v)
+	} else {
+		added = c.runAdd(v)
+	}
+	if added {
+		c.n++
+	}
+	return added
 }
 
 func (c *container) arrayAdd(v uint32) bool {
@@ -1004,7 +1075,6 @@ func (c *container) arrayAdd(v uint32) bool {
 	if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && c.array[c.n-1] < v {
 		c.unmap()
 		c.array = append(c.array, v)
-		c.n++
 		return true
 	}
 
@@ -1026,7 +1096,6 @@ func (c *container) arrayAdd(v uint32) bool {
 	c.array = append(c.array, 0)
 	copy(c.array[i+1:], c.array[i:])
 	c.array[i] = v
-	c.n++
 	return true
 }
 
@@ -1036,7 +1105,50 @@ func (c *container) bitmapAdd(v uint32) bool {
 	}
 	c.unmap()
 	c.bitmap[v/64] |= (1 << uint64(v%64))
-	c.n++
+	return true
+}
+
+func (c *container) runAdd(v uint32) bool {
+	v16 := uint16(v)
+	if len(c.runs) == 0 {
+		c.unmap()
+		c.runs = []interval16{{start: v16, last: v16}}
+		return true
+	}
+	i := 0
+	var iv interval16
+	for i, iv = range c.runs {
+		if iv.last >= v16 {
+			break
+		}
+	}
+	if v16 >= iv.start && iv.last >= v16 {
+		return false
+	}
+	c.unmap()
+	if iv.last < v16 {
+		if iv.last == v16-1 {
+			c.runs[i].last += 1
+		} else {
+			c.runs = append(c.runs, interval16{start: v16, last: v16})
+		}
+	} else if v16+1 == iv.start {
+		// combining two intervals
+		if i > 0 && c.runs[i-1].last == v16-1 {
+			c.runs[i-1].last = iv.last
+			c.runs = append(c.runs[:i], c.runs[i+1:]...)
+			return true
+		}
+		// just before an interval
+		c.runs[i].start -= 1
+	} else if i > 0 && v16-1 == c.runs[i-1].last {
+		// just after an interval
+		c.runs[i-1].last += 1
+	} else {
+		// alone
+		newIv := interval16{start: v16, last: v16}
+		c.runs = append(c.runs[:i], append([]interval16{newIv}, c.runs[i:]...)...)
+	}
 	return true
 }
 
@@ -1056,6 +1168,20 @@ func (c *container) bitmapContains(v uint32) bool {
 	return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0
 }
 
+func (c *container) runContains(v uint32) bool {
+	v16 := uint16(v)
+	for _, iv := range c.runs {
+		if v16 > iv.last {
+			continue
+		}
+		if v16 < iv.start {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
 // remove adds a value to the container.
 func (c *container) remove(v uint32) bool {
 	if c.isArray() {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index caf760c74..c9a7160cd 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -14,7 +14,138 @@
 
 package roaring
 
-import "testing"
+import (
+	"reflect"
+	"testing"
+)
+
+func TestInterval16RunLen(t *testing.T) {
+	iv := interval16{start: 7, last: 9}
+	if iv.runlen() != 3 {
+		t.Fatalf("should be 3")
+	}
+	iv = interval16{start: 7, last: 7}
+	if iv.runlen() != 1 {
+		t.Fatalf("should be 1")
+	}
+}
+
+func TestContainerRunAdd(t *testing.T) {
+	c := container{runs: make([]interval16, 0)}
+	tests := []struct {
+		op  uint32
+		exp []interval16
+	}{
+		{1, []interval16{{start: 1, last: 1}}},
+		{2, []interval16{{start: 1, last: 2}}},
+		{4, []interval16{{start: 1, last: 2}, {start: 4, last: 4}}},
+		{3, []interval16{{start: 1, last: 4}}},
+		{10, []interval16{{start: 1, last: 4}, {start: 10, last: 10}}},
+		{7, []interval16{{start: 1, last: 4}, {start: 7, last: 7}, {start: 10, last: 10}}},
+		{6, []interval16{{start: 1, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
+		{0, []interval16{{start: 0, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
+		{8, []interval16{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}},
+	}
+	for _, test := range tests {
+		ret := c.add(test.op)
+		if !ret {
+			t.Fatalf("result of adding new bit should be true: %v", c.runs)
+		}
+		if !reflect.DeepEqual(c.runs, test.exp) {
+			t.Fatalf("Should have %v, but got %v after adding %v", test.exp, c.runs, test.op)
+		}
+	}
+}
+
+func TestContainerRunAdd2(t *testing.T) {
+	c := container{runs: make([]interval16, 0)}
+	ret := c.add(0)
+	if !ret {
+		t.Fatalf("result of adding new bit should be true: %v", c.runs)
+	}
+	if !reflect.DeepEqual(c.runs, []interval16{{start: 0, last: 0}}) {
+		t.Fatalf("should have 1 run of length 1, but have %v", c.runs)
+	}
+	ret = c.add(0)
+	if ret {
+		t.Fatalf("result of adding existing bit should be false: %v", c.runs)
+	}
+}
+
+func TestRunCountRange(t *testing.T) {
+	c := container{runs: make([]interval16, 0)}
+	cnt := c.runCountRange(2, 9)
+	if cnt != 0 {
+		t.Fatalf("should get 0 from empty container, but got: %v", cnt)
+	}
+	c.add(5)
+	c.add(6)
+	c.add(7)
+
+	cnt = c.runCountRange(2, 9)
+	if cnt != 3 {
+		t.Fatalf("should get 3 from interval within range, but got: %v", cnt)
+	}
+
+	c.add(8)
+	c.add(9)
+	c.add(10)
+	c.add(11)
+
+	cnt = c.runCountRange(6, 8)
+	if cnt != 2 {
+		t.Fatalf("should get 2 from range within interval, but got: %v", cnt)
+	}
+
+	cnt = c.runCountRange(3, 9)
+	if cnt != 4 {
+		t.Fatalf("should get 4 from range overlaps front of interval, but got: %v", cnt)
+	}
+
+	cnt = c.runCountRange(9, 14)
+	if cnt != 3 {
+		t.Fatalf("should get 3 from range overlaps back of interval, but got: %v", cnt)
+	}
+
+	c.add(17)
+	c.add(18)
+	c.add(19)
+
+	cnt = c.runCountRange(1, 22)
+	if cnt != 10 {
+		t.Fatalf("should get 10 from multiple ranges in interval, but got: %v", cnt)
+	}
+
+	c.add(13)
+	c.add(14)
+
+	cnt = c.runCountRange(6, 18)
+	if cnt != 9 {
+		t.Fatalf("should get 9 from multiple ranges overlapping both sides, but got: %v", cnt)
+	}
+}
+
+func TestRunContains(t *testing.T) {
+	c := container{runs: make([]interval16, 0)}
+	if c.runContains(5) {
+		t.Fatalf("empty run container should not contain 5")
+	}
+	c.add(5)
+	if !c.runContains(5) {
+		t.Fatalf("run container with 5 should contain 5")
+	}
+
+	c.add(6)
+	c.add(7)
+
+	c.add(9)
+	c.add(10)
+	c.add(11)
+
+	if !c.runContains(10) {
+		t.Fatalf("run container with 10 in second run should contain 10")
+	}
+}
 
 func TestBitmapCountRange(t *testing.T) {
 	c := container{}
@@ -38,6 +169,12 @@ func TestBitmapCountRange(t *testing.T) {
 			t.Fatalf("test #%v count of %v from %v to %v should be %v but got %v", i, test.bitmap, test.start, test.end, test.exp, ret)
 		}
 	}
+
+	c = container{bitmap: []uint64{0xF0000001, 0xFF000000, 0xFF00000000000000}}
+	cnt := c.bitmapCountRange(62, 129)
+	if cnt != 10 {
+		t.Fatalf("count of %v from 62 to 129 should be 10, but got %v", c.bitmap, cnt)
+	}
 }
 
 func TestIntersectionCountArrayBitmap2(t *testing.T) {

From 72b6eb45e56d3875118fb3ba5eabc010aa49c1c3 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Wed, 17 May 2017 13:47:45 -0500
Subject: [PATCH 002/100] add runRemove

---
 roaring/roaring.go               | 43 ++++++++++++++++++++++++++++----
 roaring/roaring_internal_test.go | 38 ++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+), 5 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 743513562..3eca20357 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1156,8 +1156,11 @@ func (c *container) runAdd(v uint32) bool {
 func (c *container) contains(v uint32) bool {
 	if c.isArray() {
 		return c.arrayContains(v)
+	} else if c.isBitmap() {
+		return c.bitmapContains(v)
+	} else {
+		return c.runContains(v)
 	}
-	return c.bitmapContains(v)
 }
 
 func (c *container) arrayContains(v uint32) bool {
@@ -1183,11 +1186,18 @@ func (c *container) runContains(v uint32) bool {
 }
 
 // remove adds a value to the container.
-func (c *container) remove(v uint32) bool {
+func (c *container) remove(v uint32) (removed bool) {
 	if c.isArray() {
-		return c.arrayRemove(v)
+		removed = c.arrayRemove(v)
+	} else if c.isBitmap() {
+		removed = c.bitmapRemove(v)
+	} else {
+		removed = c.runRemove(v)
 	}
-	return c.bitmapRemove(v)
+	if removed {
+		c.n--
+	}
+	return removed
 }
 
 func (c *container) arrayRemove(v uint32) bool {
@@ -1197,7 +1207,6 @@ func (c *container) arrayRemove(v uint32) bool {
 	}
 	c.unmap()
 
-	c.n--
 	c.array = append(c.array[:i], c.array[i+1:]...)
 	return true
 }
@@ -1219,6 +1228,30 @@ func (c *container) bitmapRemove(v uint32) bool {
 	return true
 }
 
+func (c *container) runRemove(v uint32) bool {
+	v16 := uint16(v)
+	for i, iv := range c.runs {
+		if v16 <= iv.last {
+			if v16 < iv.start {
+				return false
+			}
+			c.unmap()
+			if v16 == iv.last && v16 == iv.start {
+				c.runs = append(c.runs[:i], c.runs[i+1:]...)
+			} else if v16 == iv.last {
+				c.runs[i].last -= 1
+			} else if v16 == iv.start {
+				c.runs[i].start += 1
+			} else if v16 > iv.start {
+				c.runs[i].last = v16 - 1
+				c.runs = append(c.runs[:i+1], append([]interval16{{start: v16 + 1, last: iv.last}}, c.runs[i+1:]...)...)
+			}
+			return true
+		}
+	}
+	return false
+}
+
 // max returns the maximum value in the container.
 func (c *container) max() uint32 {
 	if c.isArray() {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index c9a7160cd..c5736ea11 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -47,6 +47,7 @@ func TestContainerRunAdd(t *testing.T) {
 		{8, []interval16{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}},
 	}
 	for _, test := range tests {
+		c.mapped = true
 		ret := c.add(test.op)
 		if !ret {
 			t.Fatalf("result of adding new bit should be true: %v", c.runs)
@@ -54,6 +55,9 @@ func TestContainerRunAdd(t *testing.T) {
 		if !reflect.DeepEqual(c.runs, test.exp) {
 			t.Fatalf("Should have %v, but got %v after adding %v", test.exp, c.runs, test.op)
 		}
+		if c.mapped {
+			t.Fatalf("container should not be mapped after adding bit %v", test.op)
+		}
 	}
 }
 
@@ -221,3 +225,37 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
 		}
 	}
 }
+
+func TestRunRemove(t *testing.T) {
+	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	tests := []struct {
+		op     uint32
+		exp    []interval16
+		expRet bool
+	}{
+		{2, []interval16{{start: 3, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
+		{10, []interval16{{start: 3, last: 9}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
+		{12, []interval16{{start: 3, last: 9}, {start: 13, last: 13}, {start: 15, last: 16}}, true},
+		{13, []interval16{{start: 3, last: 9}, {start: 15, last: 16}}, true},
+		{16, []interval16{{start: 3, last: 9}, {start: 15, last: 15}}, true},
+		{6, []interval16{{start: 3, last: 5}, {start: 7, last: 9}, {start: 15, last: 15}}, true},
+		{8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, true},
+		{8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
+		{1, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
+		{44, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
+	}
+
+	for _, test := range tests {
+		c.mapped = true
+		ret := c.remove(test.op)
+		if ret != test.expRet || !reflect.DeepEqual(c.runs, test.exp) {
+			t.Fatalf("Unexpected result removing %v from runs. Expected %v, got %v. Expected %v, got %v", test.op, test.expRet, ret, test.exp, c.runs)
+		}
+		if ret && c.mapped {
+			t.Fatalf("container was not unmapped although bit %v was removed", test.op)
+		}
+		if !ret && !c.mapped {
+			t.Fatalf("container was unmapped although bit %v was not removed", test.op)
+		}
+	}
+}

From e80812cb0a8c835a57373d040a7e946b38a93afd Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Wed, 17 May 2017 14:10:31 -0500
Subject: [PATCH 003/100] implement runMax

---
 roaring/roaring.go               | 14 ++++++++++++--
 roaring/roaring_internal_test.go | 14 ++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 3eca20357..c610d1a0a 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1256,13 +1256,16 @@ func (c *container) runRemove(v uint32) bool {
 func (c *container) max() uint32 {
 	if c.isArray() {
 		return c.arrayMax()
+	} else if c.isBitmap() {
+		return c.bitmapMax()
+	} else {
+		return c.runMax()
 	}
-	return c.bitmapMax()
 }
 
 func (c *container) arrayMax() uint32 {
 	if len(c.array) == 0 {
-		return 0 //probably hiding some ugly bug but it prevents a crash
+		return 0 // probably hiding some ugly bug but it prevents a crash
 	}
 	return c.array[len(c.array)-1]
 }
@@ -1286,6 +1289,13 @@ func (c *container) bitmapMax() uint32 {
 	return 0
 }
 
+func (c *container) runMax() uint32 {
+	if len(c.runs) == 0 {
+		return 0
+	}
+	return uint32(c.runs[len(c.runs)-1].last)
+}
+
 // convertToArray converts the values in the bitmap to array values.
 func (c *container) convertToArray() {
 	c.array = make([]uint32, 0, c.n)
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index c5736ea11..9a0064725 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -259,3 +259,17 @@ func TestRunRemove(t *testing.T) {
 		}
 	}
 }
+
+func TestRunMax(t *testing.T) {
+	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	max := c.max()
+	if max != 16 {
+		t.Fatalf("max for %v should be 16", c.runs)
+	}
+
+	c = container{runs: []interval16{}}
+	max = c.max()
+	if max != 0 {
+		t.Fatalf("max for %v should be 0", c.runs)
+	}
+}

From 98c8fd1ded45ef830ab47f770d8e4860447c00d6 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Wed, 17 May 2017 16:18:56 -0500
Subject: [PATCH 004/100] rename conversion funcs to prepare for rle

also support rle in clone()
---
 roaring/roaring.go | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index c610d1a0a..8a5590f39 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1086,7 +1086,7 @@ func (c *container) arrayAdd(v uint32) bool {
 
 	// Convert to a bitmap container if too many values are in an array container.
 	if c.n >= ArrayMaxSize {
-		c.convertToBitmap()
+		c.arrayToBitmap()
 		return c.bitmapAdd(v)
 	}
 
@@ -1223,7 +1223,7 @@ func (c *container) bitmapRemove(v uint32) bool {
 
 	// Convert to array if we go below the threshold.
 	if c.n == ArrayMaxSize {
-		c.convertToArray()
+		c.bitmapToArray()
 	}
 	return true
 }
@@ -1297,7 +1297,7 @@ func (c *container) runMax() uint32 {
 }
 
 // convertToArray converts the values in the bitmap to array values.
-func (c *container) convertToArray() {
+func (c *container) bitmapToArray() {
 	c.array = make([]uint32, 0, c.n)
 	for i, bitmap := range c.bitmap {
 		for bitmap != 0 {
@@ -1311,7 +1311,7 @@ func (c *container) convertToArray() {
 }
 
 // convertToBitmap converts the values in array to bitmap values.
-func (c *container) convertToBitmap() {
+func (c *container) arrayToBitmap() {
 	c.bitmap = make([]uint64, bitmapN)
 	for _, v := range c.array {
 		c.bitmap[int(v)/64] |= (uint64(1) << uint(v%64))
@@ -1334,6 +1334,11 @@ func (c *container) clone() *container {
 		copy(other.bitmap, c.bitmap)
 	}
 
+	if c.runs != nil {
+		other.runs = make([]interval16, len(c.runs))
+		copy(other.runs, c.runs)
+	}
+
 	return other
 }
 

From 95b55ee7db1242b825f03c29799123a44689498a Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Thu, 18 May 2017 10:09:16 -0500
Subject: [PATCH 005/100] use uint32 instead of uint16 for rle containers

stays consistent with the array type containers. array and rle are using uint32
in case we want to change the size of roaring containers as an optimization.
This has been done in the past
https://github.com/pilosa/pilosa/commit/a8feb985e15823bd461b841f7897f2060cdf284f

The downside is that more storage is used, so we may want to convert to uint16
until we decide to play with that optimization again, but for now I'll keep the
rle implementation in sync with the array implementation.
---
 roaring/roaring.go               | 82 +++++++++++++++-----------------
 roaring/roaring_internal_test.go | 64 ++++++++++++-------------
 2 files changed, 71 insertions(+), 75 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 8a5590f39..18dcd4c88 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -902,22 +902,22 @@ type container struct {
 	n      int      // number of integers in container
 	array  []uint32 // used for array containers
 	bitmap []uint64 // used for bitmap containers
-	runs   []interval16
+	runs   []interval32
 	mapped bool // mapped directly to a byte slice when true
 }
 
-type interval16 struct {
-	start uint16
-	last  uint16
+type interval32 struct {
+	start uint32
+	last  uint32
 }
 
 // runlen returns the count of integers in the interval.
-func (iv interval16) runlen() int {
+func (iv interval32) runlen() int {
 	return 1 + int(iv.last) - int(iv.start)
 }
 
 // String produces a human viewable string of the contents.
-func (iv interval16) String() string {
+func (iv interval32) String() string {
 	return fmt.Sprintf("[%d, %d]", iv.start, iv.last)
 }
 
@@ -959,7 +959,7 @@ func (c *container) unmap() {
 		c.bitmap = tmp
 	}
 	if c.runs != nil {
-		tmp := make([]interval16, len(c.runs))
+		tmp := make([]interval32, len(c.runs))
 		copy(tmp, c.runs)
 		c.runs = tmp
 	}
@@ -1025,31 +1025,30 @@ func (c *container) bitmapCountRange(start, end uint32) int {
 }
 
 func (c *container) runCountRange(start, end uint32) (n int) {
-	start16, end16 := uint16(start), uint16(end)
 	for _, iv := range c.runs {
 		// iv is before range
-		if iv.last < start16 {
+		if iv.last < start {
 			continue
 		}
 		// iv is after range
-		if end16 < iv.start {
+		if end < iv.start {
 			break
 		}
 		// iv is superset of range
-		if iv.start < start16 && iv.last > end16 {
-			return int(end16 - start16)
+		if iv.start < start && iv.last > end {
+			return int(end - start)
 		}
 		// iv is subset of range
-		if iv.start >= start16 && iv.last < end16 {
+		if iv.start >= start && iv.last < end {
 			n += iv.runlen()
 		}
 		// iv overlaps beginning of range
-		if iv.start < start16 && iv.last < end16 {
-			n += int(iv.last - start16 + 1)
+		if iv.start < start && iv.last < end {
+			n += int(iv.last - start + 1)
 		}
 		// iv overlaps end of range
-		if iv.start > start16 && iv.last >= end16 {
-			n += int(end16 - iv.start)
+		if iv.start > start && iv.last >= end {
+			n += int(end - iv.start)
 		}
 	}
 	return n
@@ -1109,45 +1108,44 @@ func (c *container) bitmapAdd(v uint32) bool {
 }
 
 func (c *container) runAdd(v uint32) bool {
-	v16 := uint16(v)
 	if len(c.runs) == 0 {
 		c.unmap()
-		c.runs = []interval16{{start: v16, last: v16}}
+		c.runs = []interval32{{start: v, last: v}}
 		return true
 	}
 	i := 0
-	var iv interval16
+	var iv interval32
 	for i, iv = range c.runs {
-		if iv.last >= v16 {
+		if iv.last >= v {
 			break
 		}
 	}
-	if v16 >= iv.start && iv.last >= v16 {
+	if v >= iv.start && iv.last >= v {
 		return false
 	}
 	c.unmap()
-	if iv.last < v16 {
-		if iv.last == v16-1 {
+	if iv.last < v {
+		if iv.last == v-1 {
 			c.runs[i].last += 1
 		} else {
-			c.runs = append(c.runs, interval16{start: v16, last: v16})
+			c.runs = append(c.runs, interval32{start: v, last: v})
 		}
-	} else if v16+1 == iv.start {
+	} else if v+1 == iv.start {
 		// combining two intervals
-		if i > 0 && c.runs[i-1].last == v16-1 {
+		if i > 0 && c.runs[i-1].last == v-1 {
 			c.runs[i-1].last = iv.last
 			c.runs = append(c.runs[:i], c.runs[i+1:]...)
 			return true
 		}
 		// just before an interval
 		c.runs[i].start -= 1
-	} else if i > 0 && v16-1 == c.runs[i-1].last {
+	} else if i > 0 && v-1 == c.runs[i-1].last {
 		// just after an interval
 		c.runs[i-1].last += 1
 	} else {
 		// alone
-		newIv := interval16{start: v16, last: v16}
-		c.runs = append(c.runs[:i], append([]interval16{newIv}, c.runs[i:]...)...)
+		newIv := interval32{start: v, last: v}
+		c.runs = append(c.runs[:i], append([]interval32{newIv}, c.runs[i:]...)...)
 	}
 	return true
 }
@@ -1172,12 +1170,11 @@ func (c *container) bitmapContains(v uint32) bool {
 }
 
 func (c *container) runContains(v uint32) bool {
-	v16 := uint16(v)
 	for _, iv := range c.runs {
-		if v16 > iv.last {
+		if v > iv.last {
 			continue
 		}
-		if v16 < iv.start {
+		if v < iv.start {
 			return false
 		}
 		return true
@@ -1229,22 +1226,21 @@ func (c *container) bitmapRemove(v uint32) bool {
 }
 
 func (c *container) runRemove(v uint32) bool {
-	v16 := uint16(v)
 	for i, iv := range c.runs {
-		if v16 <= iv.last {
-			if v16 < iv.start {
+		if v <= iv.last {
+			if v < iv.start {
 				return false
 			}
 			c.unmap()
-			if v16 == iv.last && v16 == iv.start {
+			if v == iv.last && v == iv.start {
 				c.runs = append(c.runs[:i], c.runs[i+1:]...)
-			} else if v16 == iv.last {
+			} else if v == iv.last {
 				c.runs[i].last -= 1
-			} else if v16 == iv.start {
+			} else if v == iv.start {
 				c.runs[i].start += 1
-			} else if v16 > iv.start {
-				c.runs[i].last = v16 - 1
-				c.runs = append(c.runs[:i+1], append([]interval16{{start: v16 + 1, last: iv.last}}, c.runs[i+1:]...)...)
+			} else if v > iv.start {
+				c.runs[i].last = v - 1
+				c.runs = append(c.runs[:i+1], append([]interval32{{start: v + 1, last: iv.last}}, c.runs[i+1:]...)...)
 			}
 			return true
 		}
@@ -1335,7 +1331,7 @@ func (c *container) clone() *container {
 	}
 
 	if c.runs != nil {
-		other.runs = make([]interval16, len(c.runs))
+		other.runs = make([]interval32, len(c.runs))
 		copy(other.runs, c.runs)
 	}
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 9a0064725..37ba9ca62 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -19,32 +19,32 @@ import (
 	"testing"
 )
 
-func TestInterval16RunLen(t *testing.T) {
-	iv := interval16{start: 7, last: 9}
+func TestInterval32RunLen(t *testing.T) {
+	iv := interval32{start: 7, last: 9}
 	if iv.runlen() != 3 {
 		t.Fatalf("should be 3")
 	}
-	iv = interval16{start: 7, last: 7}
+	iv = interval32{start: 7, last: 7}
 	if iv.runlen() != 1 {
 		t.Fatalf("should be 1")
 	}
 }
 
 func TestContainerRunAdd(t *testing.T) {
-	c := container{runs: make([]interval16, 0)}
+	c := container{runs: make([]interval32, 0)}
 	tests := []struct {
 		op  uint32
-		exp []interval16
+		exp []interval32
 	}{
-		{1, []interval16{{start: 1, last: 1}}},
-		{2, []interval16{{start: 1, last: 2}}},
-		{4, []interval16{{start: 1, last: 2}, {start: 4, last: 4}}},
-		{3, []interval16{{start: 1, last: 4}}},
-		{10, []interval16{{start: 1, last: 4}, {start: 10, last: 10}}},
-		{7, []interval16{{start: 1, last: 4}, {start: 7, last: 7}, {start: 10, last: 10}}},
-		{6, []interval16{{start: 1, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
-		{0, []interval16{{start: 0, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
-		{8, []interval16{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}},
+		{1, []interval32{{start: 1, last: 1}}},
+		{2, []interval32{{start: 1, last: 2}}},
+		{4, []interval32{{start: 1, last: 2}, {start: 4, last: 4}}},
+		{3, []interval32{{start: 1, last: 4}}},
+		{10, []interval32{{start: 1, last: 4}, {start: 10, last: 10}}},
+		{7, []interval32{{start: 1, last: 4}, {start: 7, last: 7}, {start: 10, last: 10}}},
+		{6, []interval32{{start: 1, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
+		{0, []interval32{{start: 0, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
+		{8, []interval32{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}},
 	}
 	for _, test := range tests {
 		c.mapped = true
@@ -62,12 +62,12 @@ func TestContainerRunAdd(t *testing.T) {
 }
 
 func TestContainerRunAdd2(t *testing.T) {
-	c := container{runs: make([]interval16, 0)}
+	c := container{runs: make([]interval32, 0)}
 	ret := c.add(0)
 	if !ret {
 		t.Fatalf("result of adding new bit should be true: %v", c.runs)
 	}
-	if !reflect.DeepEqual(c.runs, []interval16{{start: 0, last: 0}}) {
+	if !reflect.DeepEqual(c.runs, []interval32{{start: 0, last: 0}}) {
 		t.Fatalf("should have 1 run of length 1, but have %v", c.runs)
 	}
 	ret = c.add(0)
@@ -77,7 +77,7 @@ func TestContainerRunAdd2(t *testing.T) {
 }
 
 func TestRunCountRange(t *testing.T) {
-	c := container{runs: make([]interval16, 0)}
+	c := container{runs: make([]interval32, 0)}
 	cnt := c.runCountRange(2, 9)
 	if cnt != 0 {
 		t.Fatalf("should get 0 from empty container, but got: %v", cnt)
@@ -130,7 +130,7 @@ func TestRunCountRange(t *testing.T) {
 }
 
 func TestRunContains(t *testing.T) {
-	c := container{runs: make([]interval16, 0)}
+	c := container{runs: make([]interval32, 0)}
 	if c.runContains(5) {
 		t.Fatalf("empty run container should not contain 5")
 	}
@@ -227,22 +227,22 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
 }
 
 func TestRunRemove(t *testing.T) {
-	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	c := container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
 	tests := []struct {
 		op     uint32
-		exp    []interval16
+		exp    []interval32
 		expRet bool
 	}{
-		{2, []interval16{{start: 3, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
-		{10, []interval16{{start: 3, last: 9}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
-		{12, []interval16{{start: 3, last: 9}, {start: 13, last: 13}, {start: 15, last: 16}}, true},
-		{13, []interval16{{start: 3, last: 9}, {start: 15, last: 16}}, true},
-		{16, []interval16{{start: 3, last: 9}, {start: 15, last: 15}}, true},
-		{6, []interval16{{start: 3, last: 5}, {start: 7, last: 9}, {start: 15, last: 15}}, true},
-		{8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, true},
-		{8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
-		{1, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
-		{44, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
+		{2, []interval32{{start: 3, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
+		{10, []interval32{{start: 3, last: 9}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
+		{12, []interval32{{start: 3, last: 9}, {start: 13, last: 13}, {start: 15, last: 16}}, true},
+		{13, []interval32{{start: 3, last: 9}, {start: 15, last: 16}}, true},
+		{16, []interval32{{start: 3, last: 9}, {start: 15, last: 15}}, true},
+		{6, []interval32{{start: 3, last: 5}, {start: 7, last: 9}, {start: 15, last: 15}}, true},
+		{8, []interval32{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, true},
+		{8, []interval32{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
+		{1, []interval32{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
+		{44, []interval32{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
 	}
 
 	for _, test := range tests {
@@ -261,13 +261,13 @@ func TestRunRemove(t *testing.T) {
 }
 
 func TestRunMax(t *testing.T) {
-	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	c := container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
 	max := c.max()
 	if max != 16 {
 		t.Fatalf("max for %v should be 16", c.runs)
 	}
 
-	c = container{runs: []interval16{}}
+	c = container{runs: []interval32{}}
 	max = c.max()
 	if max != 0 {
 		t.Fatalf("max for %v should be 0", c.runs)

From 014005b5d55f3c67d124eae523406e3e507746d2 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Thu, 18 May 2017 10:39:45 -0500
Subject: [PATCH 006/100] when choosing between container types, keep bitmap as
 default

---
 roaring/roaring.go | 50 ++++++++++++++++++++++++++++++----------------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 18dcd4c88..d0429a08c 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -975,10 +975,10 @@ func (c *container) count() (n int) {
 func (c *container) countRange(start, end uint32) (n int) {
 	if c.isArray() {
 		return c.arrayCountRange(start, end)
-	} else if c.isBitmap() {
-		return c.bitmapCountRange(start, end)
+	} else if c.isRun() {
+		return c.runCountRange(start, end)
 	}
-	return c.runCountRange(start, end)
+	return c.bitmapCountRange(start, end)
 }
 
 func (c *container) arrayCountRange(start, end uint32) (n int) {
@@ -1058,10 +1058,10 @@ func (c *container) runCountRange(start, end uint32) (n int) {
 func (c *container) add(v uint32) (added bool) {
 	if c.isArray() {
 		added = c.arrayAdd(v)
-	} else if c.isBitmap() {
-		added = c.bitmapAdd(v)
-	} else {
+	} else if c.isRun() {
 		added = c.runAdd(v)
+	} else {
+		added = c.bitmapAdd(v)
 	}
 	if added {
 		c.n++
@@ -1154,10 +1154,10 @@ func (c *container) runAdd(v uint32) bool {
 func (c *container) contains(v uint32) bool {
 	if c.isArray() {
 		return c.arrayContains(v)
-	} else if c.isBitmap() {
-		return c.bitmapContains(v)
-	} else {
+	} else if c.isRun() {
 		return c.runContains(v)
+	} else {
+		return c.bitmapContains(v)
 	}
 }
 
@@ -1186,10 +1186,10 @@ func (c *container) runContains(v uint32) bool {
 func (c *container) remove(v uint32) (removed bool) {
 	if c.isArray() {
 		removed = c.arrayRemove(v)
-	} else if c.isBitmap() {
-		removed = c.bitmapRemove(v)
-	} else {
+	} else if c.isRun() {
 		removed = c.runRemove(v)
+	} else {
+		removed = c.bitmapRemove(v)
 	}
 	if removed {
 		c.n--
@@ -1252,10 +1252,10 @@ func (c *container) runRemove(v uint32) bool {
 func (c *container) max() uint32 {
 	if c.isArray() {
 		return c.arrayMax()
-	} else if c.isBitmap() {
-		return c.bitmapMax()
-	} else {
+	} else if c.isRun() {
 		return c.runMax()
+	} else {
+		return c.bitmapMax()
 	}
 }
 
@@ -1342,8 +1342,11 @@ func (c *container) clone() *container {
 func (c *container) WriteTo(w io.Writer) (n int64, err error) {
 	if c.isArray() {
 		return c.arrayWriteTo(w)
+	} else if c.isRun() {
+		return c.runWriteTo(w)
+	} else {
+		return c.bitmapWriteTo(w)
 	}
-	return c.bitmapWriteTo(w)
 }
 
 func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) {
@@ -1366,12 +1369,20 @@ func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) {
 	return int64(nn), err
 }
 
+func (c *container) runWriteTo(w io.Writer) (n int64, err error) {
+	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:8*c.n])
+	return int64(nn), err
+}
+
 // size returns the encoded size of the container, in bytes.
 func (c *container) size() int {
 	if c.isArray() {
 		return len(c.array) * 4
+	} else if c.isRun() {
+		return len(c.runs) * 8
+	} else {
+		return len(c.bitmap) * 8
 	}
-	return len(c.bitmap) * 8
 }
 
 // info returns the current stats about the container.
@@ -1381,6 +1392,9 @@ func (c *container) info() ContainerInfo {
 	if c.isArray() {
 		info.Type = "array"
 		info.Alloc = len(c.array) * 4
+	} else if c.isRun() {
+		info.Type = "run"
+		info.Alloc = len(c.runs) * 8
 	} else {
 		info.Type = "bitmap"
 		info.Alloc = len(c.bitmap) * 8
@@ -1389,6 +1403,8 @@ func (c *container) info() ContainerInfo {
 	if c.mapped {
 		if c.isArray() {
 			info.Pointer = unsafe.Pointer(&c.array[0])
+		} else if c.isRun() {
+			info.Pointer = unsafe.Pointer(&c.runs[0])
 		} else {
 			info.Pointer = unsafe.Pointer(&c.bitmap[0])
 		}

From 20cec1363b86a8f360b82a6c32ee10059826c416 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Thu, 18 May 2017 15:40:03 -0500
Subject: [PATCH 007/100] implement intersectionCounts for run containers

---
 roaring/roaring.go               | 71 ++++++++++++++++++++++++++++-
 roaring/roaring_internal_test.go | 77 ++++++++++++++++++++++++++++++++
 2 files changed, 146 insertions(+), 2 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index d0429a08c..09d173ede 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1446,12 +1446,24 @@ func intersectionCount(a, b *container) uint64 {
 	if a.isArray() {
 		if b.isArray() {
 			return intersectionCountArrayArray(a, b)
+		} else if b.isRun() {
+			return intersectionCountArrayRun(a, b)
 		} else {
 			return intersectionCountArrayBitmap(a, b)
 		}
+	} else if a.isRun() {
+		if b.isArray() {
+			return intersectionCountArrayRun(b, a)
+		} else if b.isRun() {
+			return intersectionCountRunRun(a, b)
+		} else {
+			return intersectionCountBitmapRun(b, a)
+		}
 	} else {
 		if b.isArray() {
 			return intersectionCountArrayBitmap(b, a)
+		} else if b.isRun() {
+			return intersectionCountBitmapRun(a, b)
 		} else {
 			return intersectionCountBitmapBitmap(a, b)
 		}
@@ -1474,6 +1486,61 @@ func intersectionCountArrayArray(a, b *container) (n uint64) {
 	return n
 }
 
+func intersectionCountArrayRun(a, b *container) (n uint64) {
+	na, nb := len(a.array), len(b.runs)
+	for i, j := 0, 0; i < na && j < nb; {
+		va, vb := a.array[i], b.runs[j]
+		if va < vb.start {
+			i++
+		} else if va >= vb.start && va <= vb.last {
+			i++
+			n++
+		} else if va > vb.last {
+			j++
+		}
+	}
+	return n
+}
+
+func intersectionCountRunRun(a, b *container) uint64 {
+	var n uint32
+	na, nb := len(a.runs), len(b.runs)
+	for i, j := 0, 0; i < na && j < nb; {
+		va, vb := a.runs[i], b.runs[j]
+		if va.last < vb.start {
+			// |--va--| |--vb--|
+			i++
+		} else if va.start > vb.last {
+			// |--vb--| |--va--|
+			j++
+		} else if va.last > vb.last && va.start >= vb.start {
+			// |--vb-|-|-va--|
+			n += 1 + vb.last - va.start
+			j++
+		} else if va.last > vb.last && va.start < vb.start {
+			// |--va|--vb--|--|
+			n += 1 + vb.last - vb.start
+			j++
+		} else if va.last <= vb.last && va.start >= vb.start {
+			// |--vb|--va--|--|
+			n += 1 + va.last - va.start
+			i++
+		} else if va.last <= vb.last && va.start < vb.start {
+			// |--va-|-|-vb--|
+			n += 1 + va.last - vb.start
+			i++
+		}
+	}
+	return uint64(n)
+}
+
+func intersectionCountBitmapRun(a, b *container) (n uint64) {
+	for _, iv := range b.runs {
+		n += uint64(a.bitmapCountRange(iv.start, iv.last+1))
+	}
+	return n
+}
+
 func intersectionCountArrayBitmapOld(a, b *container) (n uint64) {
 	// Copy array header so we can shrink it.
 	array := a.array
@@ -1878,7 +1945,7 @@ func xorArrayBitmap(a, b *container) *container {
 	}
 
 	if output.count() < ArrayMaxSize {
-		output.convertToArray()
+		output.bitmapToArray()
 	}
 
 	return output
@@ -1896,7 +1963,7 @@ func xorBitmapBitmap(a, b *container) *container {
 	}
 
 	if output.count() < ArrayMaxSize {
-		output.convertToArray()
+		output.bitmapToArray()
 	}
 	return output
 }
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 37ba9ca62..ec022305c 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -273,3 +273,80 @@ func TestRunMax(t *testing.T) {
 		t.Fatalf("max for %v should be 0", c.runs)
 	}
 }
+
+func TestIntersectionCountArrayRun(t *testing.T) {
+	a := &container{array: []uint32{1, 5, 10, 11, 12}}
+	b := &container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+
+	ret := intersectionCountArrayRun(a, b)
+	if ret != 3 {
+		t.Fatalf("count of %v with %v should be 3, but got %v", a.array, b.runs, ret)
+	}
+}
+
+func TestIntersectionCountBitmapRun(t *testing.T) {
+	a := &container{bitmap: []uint64{1}}
+	b := &container{runs: []interval32{{start: 63, last: 64}}}
+
+	ret := intersectionCountBitmapRun(a, b)
+	if ret != 1 {
+		t.Fatalf("count of %v with %v should be 1, but got %v", a.bitmap, b.runs, ret)
+	}
+
+	a = &container{bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
+	b = &container{runs: []interval32{{start: 33, last: 35}, {start: 62, last: 69}, {start: 130, last: 150}, {start: 186, last: 300}}}
+
+	ret = intersectionCountBitmapRun(a, b)
+	if ret != 22 {
+		t.Fatalf("count of %v with %v should be 22, but got %v", a.bitmap, b.runs, ret)
+	}
+}
+
+func TestIntersectionCountRunRun(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		aruns []interval32
+		bruns []interval32
+		exp   uint64
+	}{
+		{
+			aruns: []interval32{},
+			bruns: []interval32{{start: 3, last: 8}}, exp: 0},
+		{
+			aruns: []interval32{{start: 2, last: 10}},
+			bruns: []interval32{{start: 3, last: 8}}, exp: 6},
+		{
+			aruns: []interval32{{start: 2, last: 10}},
+			bruns: []interval32{{start: 1, last: 11}}, exp: 9},
+		{
+			aruns: []interval32{{start: 2, last: 10}},
+			bruns: []interval32{{start: 0, last: 2}}, exp: 1},
+		{
+			aruns: []interval32{{start: 2, last: 10}},
+			bruns: []interval32{{start: 1, last: 10}}, exp: 9},
+		{
+			aruns: []interval32{{start: 2, last: 10}},
+			bruns: []interval32{{start: 5, last: 12}}, exp: 6},
+		{
+			aruns: []interval32{{start: 2, last: 10}},
+			bruns: []interval32{{start: 10, last: 99}}, exp: 1},
+		{
+			aruns: []interval32{{start: 2, last: 10}, {start: 44, last: 99}},
+			bruns: []interval32{{start: 12, last: 14}}, exp: 0},
+		{
+			aruns: []interval32{{start: 2, last: 10}, {start: 12, last: 13}},
+			bruns: []interval32{{start: 2, last: 10}, {start: 12, last: 13}}, exp: 11},
+		{
+			aruns: []interval32{{start: 8, last: 12}, {start: 15, last: 19}},
+			bruns: []interval32{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6},
+	}
+	for i, test := range tests {
+		a.runs = test.aruns
+		b.runs = test.bruns
+		ret := intersectionCountRunRun(a, b)
+		if ret != test.exp {
+			t.Fatalf("test #%v failed intersecting %v with %v should be %v, but got %v", i, test.aruns, test.bruns, test.exp, ret)
+		}
+	}
+}

From 0c3f61a42675f704ee3b9c6fd29660941dbcd70e Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Thu, 18 May 2017 17:22:13 -0500
Subject: [PATCH 008/100] implement intersectArrayRun

---
 roaring/roaring.go               | 42 ++++++++++++++++++++++++++++++++
 roaring/roaring_internal_test.go | 40 ++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 09d173ede..1d8ef1223 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1603,12 +1603,24 @@ func intersect(a, b *container) *container {
 	if a.isArray() {
 		if b.isArray() {
 			return intersectArrayArray(a, b)
+		} else if b.isRun() {
+			return intersectArrayRun(a, b)
 		} else {
 			return intersectArrayBitmap(a, b)
 		}
+	} else if a.isRun() {
+		if b.isArray() {
+			return intersectArrayRun(b, a)
+		} else if b.isRun() {
+			return intersectRunRun(a, b)
+		} else {
+			return intersectBitmapRun(b, a)
+		}
 	} else {
 		if b.isArray() {
 			return intersectArrayBitmap(b, a)
+		} else if b.isRun() {
+			return intersectBitmapRun(a, b)
 		} else {
 			return intersectBitmapBitmap(a, b)
 		}
@@ -1633,6 +1645,36 @@ func intersectArrayArray(a, b *container) *container {
 	return output
 }
 
+func intersectArrayRun(a, b *container) *container {
+	output := &container{}
+	na, nb := len(a.array), len(b.runs)
+	for i, j := 0, 0; i < na && j < nb; {
+		va, vb := a.array[i], b.runs[j]
+		if va < vb.start {
+			i++
+		} else if va > vb.last {
+			j++
+		} else {
+			output.array = append(output.array, va)
+			output.n++
+			i++
+		}
+	}
+	return output
+}
+
+func intersectRunRun(a, b *container) *container {
+	output := &container{}
+	// TODO
+	return output
+}
+
+func intersectBitmapRun(a, b *container) *container {
+	output := &container{}
+	// TODO
+	return output
+}
+
 func intersectArrayBitmap(a, b *container) *container {
 	output := &container{}
 	itr := newBufIterator(newBitmapIterator(b.bitmap))
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index ec022305c..8f771a1fe 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -350,3 +350,43 @@ func TestIntersectionCountRunRun(t *testing.T) {
 		}
 	}
 }
+
+func TestIntersectArrayRun(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		array []uint32
+		runs  []interval32
+		exp   []uint32
+	}{
+		{
+			array: []uint32{1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval32{{start: 5, last: 10}},
+			exp:   []uint32{5, 7, 10},
+		},
+		{
+			array: []uint32{},
+			runs:  []interval32{{start: 5, last: 10}},
+			exp:   []uint32(nil),
+		},
+		{
+			array: []uint32{1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval32{},
+			exp:   []uint32(nil),
+		},
+		{
+			array: []uint32{0, 1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval32{{start: 0, last: 5}, {start: 7, last: 7}},
+			exp:   []uint32{0, 1, 4, 5, 7},
+		},
+	}
+
+	for i, test := range tests {
+		a.array = test.array
+		b.runs = test.runs
+		ret := intersectArrayRun(a, b)
+		if !reflect.DeepEqual(ret.array, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
+		}
+	}
+}

From 3e8030aee427e3de837431a54156919865bf7047 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 19 May 2017 08:22:56 -0500
Subject: [PATCH 009/100] implement intersectRunRun

---
 roaring/roaring.go               | 35 +++++++++++++++++++++++++++++++-
 roaring/roaring_internal_test.go | 35 ++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 1d8ef1223..516a76f5e 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1645,6 +1645,9 @@ func intersectArrayArray(a, b *container) *container {
 	return output
 }
 
+// intersectArrayRun computes the intersect of an array container and a run
+// container. The return is always an array container (since it's guaranteed to
+// be low-cardinality)
 func intersectArrayRun(a, b *container) *container {
 	output := &container{}
 	na, nb := len(a.array), len(b.runs)
@@ -1663,9 +1666,39 @@ func intersectArrayRun(a, b *container) *container {
 	return output
 }
 
+// intersectRunRun computes the intersect of two run containers. The output is
+// always a run container, since it can't possible have more runs than either of
+// the inputs. (note: it is possible that an array container would be better in
+// some cases, but probably not worth complicating the implementation)
 func intersectRunRun(a, b *container) *container {
 	output := &container{}
-	// TODO
+	na, nb := len(a.runs), len(b.runs)
+	for i, j := 0, 0; i < na && j < nb; {
+		va, vb := a.runs[i], b.runs[j]
+		if va.last < vb.start {
+			// |--va--| |--vb--|
+			i++
+		} else if vb.last < va.start {
+			// |--vb--| |--va--|
+			j++
+		} else if va.last > vb.last && va.start >= vb.start {
+			// |--vb-|-|-va--|
+			output.runs = append(output.runs, interval32{start: va.start, last: vb.last})
+			j++
+		} else if va.last > vb.last && va.start < vb.start {
+			// |--va|--vb--|--|
+			output.runs = append(output.runs, vb)
+			j++
+		} else if va.last <= vb.last && va.start >= vb.start {
+			// |--vb|--va--|--|
+			output.runs = append(output.runs, va)
+			i++
+		} else if va.last <= vb.last && va.start < vb.start {
+			// |--va-|-|-vb--|
+			output.runs = append(output.runs, interval32{start: vb.start, last: va.last})
+			i++
+		}
+	}
 	return output
 }
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 8f771a1fe..c250d57d7 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -390,3 +390,38 @@ func TestIntersectArrayRun(t *testing.T) {
 		}
 	}
 }
+
+func TestIntersectRunRun(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		aruns []interval32
+		bruns []interval32
+		exp   []interval32
+	}{
+		{
+			aruns: []interval32{},
+			bruns: []interval32{{start: 5, last: 10}},
+			exp:   []interval32(nil),
+		},
+		{
+			aruns: []interval32{{start: 5, last: 12}},
+			bruns: []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 5, last: 10}},
+		},
+		{
+			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			bruns: []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 10}},
+		},
+	}
+	for i, test := range tests {
+		a.runs = test.aruns
+		b.runs = test.bruns
+		ret := intersectRunRun(a, b)
+		if !reflect.DeepEqual(ret.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
+		}
+	}
+
+}

From 8954a6ae8c13dbac5705498b268a619d3599224f Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 19 May 2017 15:15:23 -0500
Subject: [PATCH 010/100] fix bitmapCountRange and test

had been thinking that index 0 was the most significant bit, but based on the
bitmapAdd function, it must be the least significant bit
---
 roaring/roaring_internal_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index c250d57d7..938c64a83 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -167,6 +167,7 @@ func TestBitmapCountRange(t *testing.T) {
 		{start: 66, end: 71, bitmap: []uint64{0xF, 0xFFFFFFFFFFFFFF18}, exp: 2},
 		{start: 63, end: 64, bitmap: []uint64{0x8000000000000000}, exp: 1},
 	}
+
 	for i, test := range tests {
 		c.bitmap = test.bitmap
 		if ret := c.bitmapCountRange(test.start, test.end); ret != test.exp {

From 250e8f6f5b704ef8fa567a9cd62696c80d97dd99 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Thu, 25 May 2017 16:53:26 -0500
Subject: [PATCH 011/100] implement intersectBitmapRun

wowza
---
 roaring/roaring.go               | 47 ++++++++++++++-
 roaring/roaring_internal_test.go | 99 ++++++++++++++++++++++++++++++++
 2 files changed, 144 insertions(+), 2 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 516a76f5e..81fce64c6 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1702,9 +1702,52 @@ func intersectRunRun(a, b *container) *container {
 	return output
 }
 
+// intersectBitmapRun returns an array container of the run container's
+// cardinality is < 4096. Otherwise it returns a bitmap container.
 func intersectBitmapRun(a, b *container) *container {
-	output := &container{}
-	// TODO
+	var output *container
+	if b.n < 4096 {
+		// output is array container
+		output = &container{}
+		for _, iv := range b.runs {
+			for i := iv.start; i <= iv.last; i++ {
+				if a.bitmapContains(i) {
+					output.array = append(output.array, i)
+				}
+			}
+		}
+	} else {
+		// right now this iterates through the runs and sets integers in the
+		// bitmap that are in the runs. alternately, we could zero out ranges in
+		// the bitmap which are between runs.
+		output = &container{
+			bitmap: make([]uint64, bitmapN),
+		}
+		for j := 0; j < len(b.runs); j++ {
+			vb := b.runs[j]
+			i := vb.start / 64 // index into a
+			vastart := 64 * i
+			valast := vastart + 63
+			for valast >= vb.start && vastart <= vb.last {
+				if vastart >= vb.start && valast <= vb.last { // a within b
+					output.bitmap[i] = a.bitmap[i]
+				} else if vb.start >= vastart && vb.last <= valast { // b within a
+					var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart)
+					output.bitmap[i] |= a.bitmap[i] & mask
+				} else if vastart < vb.start { // a overlaps front of b
+					offset := 64 - (1 + valast - vb.start)
+					output.bitmap[i] |= (a.bitmap[i] >> offset) << offset
+				} else if vb.start < vastart { // b overlaps front of a
+					offset := 64 - (1 + vb.last - vastart)
+					output.bitmap[i] |= (a.bitmap[i] << offset) >> offset
+				}
+				// update loop vars
+				i++
+				vastart = 64 * i
+				valast = vastart + 63
+			}
+		}
+	}
 	return output
 }
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 938c64a83..d31b4e237 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -426,3 +426,102 @@ func TestIntersectRunRun(t *testing.T) {
 	}
 
 }
+
+func TestIntersectBitmapRunBitmap(t *testing.T) {
+	a := &container{bitmap: make([]uint64, bitmapN)}
+	b := &container{}
+	tests := []struct {
+		bitmap []uint64
+		runs   []interval32
+		exp    []uint64
+	}{
+		{
+			bitmap: []uint64{1},
+			runs:   []interval32{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}},
+			exp:    []uint64{1},
+		},
+		{
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
+			runs:   []interval32{{start: 1, last: 1}},
+			exp:    []uint64{2},
+		},
+		{
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
+			runs:   []interval32{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}},
+			exp:    []uint64{0xe000000000001C02},
+		},
+		{
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF},
+			runs:   []interval32{{start: 1, last: 1}, {start: 61, last: 77}},
+			exp:    []uint64{0xE000000000000002, 0x00000000000003FFF},
+		},
+		{
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1},
+			runs:   []interval32{{start: 63, last: 10000}},
+			exp:    []uint64{0x8000000000000000, 1, 1, 1, 0xA, 1, 1, 0, 1},
+		},
+	}
+	for i, test := range tests {
+		for i, v := range test.bitmap {
+			a.bitmap[i] = v
+		}
+		b.runs = test.runs
+		b.n = 4097 // ;)
+		exp := make([]uint64, bitmapN)
+		for i, v := range test.exp {
+			exp[i] = v
+		}
+		ret := intersectBitmapRun(a, b)
+		if !reflect.DeepEqual(ret.bitmap, exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, exp, ret.bitmap)
+		}
+	}
+
+}
+
+func TestIntersectBitmapRunArray(t *testing.T) {
+	a := &container{bitmap: make([]uint64, bitmapN)}
+	b := &container{}
+	tests := []struct {
+		bitmap []uint64
+		runs   []interval32
+		exp    []uint32
+	}{
+		{
+			bitmap: []uint64{1},
+			runs:   []interval32{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}},
+			exp:    []uint32{0},
+		},
+		{
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
+			runs:   []interval32{{start: 1, last: 1}},
+			exp:    []uint32{1},
+		},
+		{
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
+			runs:   []interval32{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}},
+			exp:    []uint32{1, 10, 11, 12, 61, 62, 63},
+		},
+		{
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF},
+			runs:   []interval32{{start: 1, last: 1}, {start: 61, last: 68}},
+			exp:    []uint32{1, 61, 62, 63, 64, 65, 66, 67, 68},
+		},
+		{
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1},
+			runs:   []interval32{{start: 63, last: 10000}},
+			exp:    []uint32{63, 64, 128, 192, 257, 259, 320, 384, 512},
+		},
+	}
+	for i, test := range tests {
+		for i, v := range test.bitmap {
+			a.bitmap[i] = v
+		}
+		b.runs = test.runs
+		ret := intersectBitmapRun(a, b)
+		if !reflect.DeepEqual(ret.array, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
+		}
+	}
+
+}

From 9781a4c78b458261eac0eda4989e2cd29a66daac Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 26 May 2017 11:32:20 -0500
Subject: [PATCH 012/100] fix tests, prep for union rle implementations

---
 roaring/roaring.go               | 24 ++++++++++++++++++++++++
 roaring/roaring_internal_test.go |  8 ++++----
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 81fce64c6..95fcaed15 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1804,12 +1804,24 @@ func union(a, b *container) *container {
 	if a.isArray() {
 		if b.isArray() {
 			return unionArrayArray(a, b)
+		} else if b.isRun() {
+			return unionArrayRun(a, b)
 		} else {
 			return unionArrayBitmap(a, b)
 		}
+	} else if a.isRun() {
+		if b.isArray() {
+			return unionArrayRun(b, a)
+		} else if b.isRun() {
+			return unionRunRun(a, b)
+		} else {
+			return unionBitmapRun(b, a)
+		}
 	} else {
 		if b.isArray() {
 			return unionArrayBitmap(b, a)
+		} else if b.isRun() {
+			return unionBitmapRun(a, b)
 		} else {
 			return unionBitmapBitmap(a, b)
 		}
@@ -1847,6 +1859,18 @@ func unionArrayArray(a, b *container) *container {
 	return output
 }
 
+func unionArrayRun(a, b *container) *container {
+	return nil
+}
+
+func unionRunRun(a, b *container) *container {
+	return nil
+}
+
+func unionBitmapRun(a, b *container) *container {
+	return nil
+}
+
 func unionArrayBitmap(a, b *container) *container {
 	output := &container{}
 	itr := newBufIterator(newBitmapIterator(b.bitmap))
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index d31b4e237..91e256e05 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -286,7 +286,7 @@ func TestIntersectionCountArrayRun(t *testing.T) {
 }
 
 func TestIntersectionCountBitmapRun(t *testing.T) {
-	a := &container{bitmap: []uint64{1}}
+	a := &container{bitmap: []uint64{0x8000000000000000}}
 	b := &container{runs: []interval32{{start: 63, last: 64}}}
 
 	ret := intersectionCountBitmapRun(a, b)
@@ -295,11 +295,11 @@ func TestIntersectionCountBitmapRun(t *testing.T) {
 	}
 
 	a = &container{bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
-	b = &container{runs: []interval32{{start: 33, last: 35}, {start: 62, last: 69}, {start: 130, last: 150}, {start: 186, last: 300}}}
+	b = &container{runs: []interval32{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
 
 	ret = intersectionCountBitmapRun(a, b)
-	if ret != 22 {
-		t.Fatalf("count of %v with %v should be 22, but got %v", a.bitmap, b.runs, ret)
+	if ret != 14 {
+		t.Fatalf("count of %v with %v should be 14, but got %v", a.bitmap, b.runs, ret)
 	}
 }
 

From cf445f8012f484a8f637c474469bae5b21f8286c Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 26 May 2017 13:23:05 -0500
Subject: [PATCH 013/100] stub out the rest of the rle methods

---
 roaring/roaring.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 95fcaed15..d4a89b5dd 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1920,12 +1920,24 @@ func difference(a, b *container) *container {
 	if a.isArray() {
 		if b.isArray() {
 			return differenceArrayArray(a, b)
+		} else if b.isRun() {
+			return differenceArrayRun(a, b)
 		} else {
 			return differenceArrayBitmap(a, b)
 		}
+	} else if a.isRun() {
+		if b.isArray() {
+			return differenceRunArray(a, b)
+		} else if b.isRun() {
+			return differenceRunRun(a, b)
+		} else {
+			return differenceRunBitmap(a, b)
+		}
 	} else {
 		if b.isArray() {
 			return differenceBitmapArray(a, b)
+		} else if b.isRun() {
+			return differenceBitmapRun(a, b)
 		} else {
 			return differenceBitmapBitmap(a, b)
 		}
@@ -1956,6 +1968,27 @@ func differenceArrayArray(a, b *container) *container {
 	return output
 }
 
+func differenceArrayRun(a, b *container) *container {
+	// TODO
+	return nil
+}
+func differenceRunArray(a, b *container) *container {
+	// TODO
+	return nil
+}
+func differenceRunBitmap(a, b *container) *container {
+	// TODO
+	return nil
+}
+func differenceBitmapRun(a, b *container) *container {
+	// TODO
+	return nil
+}
+func differenceRunRun(a, b *container) *container {
+	// TODO
+	return nil
+}
+
 func differenceArrayBitmap(a, b *container) *container {
 	output := &container{}
 	itr := newBufIterator(newBitmapIterator(b.bitmap))

From 24c03f07f6ab0163f04801473f4dc906cd3583fc Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Fri, 26 May 2017 13:23:29 -0500
Subject: [PATCH 014/100] add note to improve intersectRunRun func

---
 roaring/roaring.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index d4a89b5dd..729cf3ec4 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1671,6 +1671,7 @@ func intersectArrayRun(a, b *container) *container {
 // the inputs. (note: it is possible that an array container would be better in
 // some cases, but probably not worth complicating the implementation)
 func intersectRunRun(a, b *container) *container {
+	// TODO need to take the last element of output.runs into account during each loop - possible that instead of appending, the last run should just be expanded
 	output := &container{}
 	na, nb := len(a.runs), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {

From 4848a99acaf1e7cd5db2c7c9f6cddda01deac4c1 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 30 May 2017 13:19:02 -0500
Subject: [PATCH 015/100] unionRunRun working - messy

---
 roaring/roaring.go               | 66 +++++++++++++++++++++++++++++++-
 roaring/roaring_internal_test.go | 35 +++++++++++++++++
 2 files changed, 100 insertions(+), 1 deletion(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 729cf3ec4..201e2be21 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1865,7 +1865,71 @@ func unionArrayRun(a, b *container) *container {
 }
 
 func unionRunRun(a, b *container) *container {
-	return nil
+	na, nb := len(a.runs), len(b.runs)
+	if na == 0 {
+		return b.clone()
+	} else if nb == 0 {
+		return a.clone()
+	}
+	output := &container{
+		runs: make([]interval32, 0, na+nb),
+	}
+	i, j := 0, 0
+	if a.runs[0].start < b.runs[0].start {
+		output.runs = append(output.runs, a.runs[0])
+		i = 1
+	} else {
+		output.runs = append(output.runs, b.runs[0])
+		j = 1
+	}
+	for i < na && j < nb {
+		va, vb := a.runs[i], b.runs[j]
+		lastrun := output.runs[len(output.runs)-1]
+		if va.start < vb.start {
+			if va.start <= lastrun.last+1 {
+				if lastrun.last < va.last {
+					lastrun.last = va.last
+					output.runs[len(output.runs)-1] = lastrun
+				}
+			} else {
+				output.runs = append(output.runs, va)
+			}
+			i++
+		} else {
+			if vb.start <= lastrun.last+1 {
+				if lastrun.last < vb.last {
+					lastrun.last = vb.last
+					output.runs[len(output.runs)-1] = lastrun
+				}
+			} else {
+				output.runs = append(output.runs, vb)
+			}
+			j++
+		}
+	}
+	var k int
+	var rest []interval32
+	if i < na {
+		k = i
+		rest = a.runs
+	} else {
+		k = j
+		rest = b.runs
+	}
+	for k < len(rest) {
+		vk := rest[k]
+		lastrun := output.runs[len(output.runs)-1]
+		if vk.start <= lastrun.last+1 {
+			if lastrun.last < vk.last {
+				lastrun.last = vk.last
+				output.runs[len(output.runs)-1] = lastrun
+			}
+		} else {
+			output.runs = append(output.runs, vk)
+		}
+		k++
+	}
+	return output
 }
 
 func unionBitmapRun(a, b *container) *container {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 91e256e05..0231bf93f 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -525,3 +525,38 @@ func TestIntersectBitmapRunArray(t *testing.T) {
 	}
 
 }
+
+func TestUnionRunRun(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		aruns []interval32
+		bruns []interval32
+		exp   []interval32
+	}{
+		{
+			aruns: []interval32{},
+			bruns: []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 5, last: 10}},
+		},
+		{
+			aruns: []interval32{{start: 5, last: 12}},
+			bruns: []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 5, last: 12}},
+		},
+		{
+			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			bruns: []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 1, last: 3}, {start: 5, last: 12}},
+		},
+	}
+	for i, test := range tests {
+		a.runs = test.aruns
+		b.runs = test.bruns
+		ret := unionRunRun(a, b)
+		if !reflect.DeepEqual(ret.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
+		}
+	}
+
+}

From c5a0f23257e13d6829dc3ba07f6294f3faf559ce Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 30 May 2017 15:05:08 -0500
Subject: [PATCH 016/100] clean up unionRunRun

---
 roaring/roaring.go               | 76 +++++++++-----------------------
 roaring/roaring_internal_test.go | 26 ++++++++++-
 2 files changed, 47 insertions(+), 55 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 201e2be21..c032a83cd 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1862,73 +1862,41 @@ func unionArrayArray(a, b *container) *container {
 
 func unionArrayRun(a, b *container) *container {
 	return nil
+func (c *container) runAppendInterval(v interval32) {
+	if len(c.runs) == 0 {
+		c.runs = append(c.runs, v)
+	} else {
+		lastidx := len(c.runs) - 1
+		if c.runs[lastidx].last+1 >= v.start && v.last > c.runs[lastidx].last {
+			c.runs[lastidx].last = v.last
+		} else if c.runs[lastidx].last+1 < v.start {
+			c.runs = append(c.runs, v)
+		}
+	}
 }
 
 func unionRunRun(a, b *container) *container {
 	na, nb := len(a.runs), len(b.runs)
-	if na == 0 {
-		return b.clone()
-	} else if nb == 0 {
-		return a.clone()
-	}
 	output := &container{
 		runs: make([]interval32, 0, na+nb),
 	}
 	i, j := 0, 0
-	if a.runs[0].start < b.runs[0].start {
-		output.runs = append(output.runs, a.runs[0])
-		i = 1
-	} else {
-		output.runs = append(output.runs, b.runs[0])
-		j = 1
-	}
-	for i < na && j < nb {
-		va, vb := a.runs[i], b.runs[j]
-		lastrun := output.runs[len(output.runs)-1]
-		if va.start < vb.start {
-			if va.start <= lastrun.last+1 {
-				if lastrun.last < va.last {
-					lastrun.last = va.last
-					output.runs[len(output.runs)-1] = lastrun
-				}
-			} else {
-				output.runs = append(output.runs, va)
-			}
+	var va, vb interval32
+	for i < na || j < nb {
+		if i < na {
+			va = a.runs[i]
+		}
+		if j < nb {
+			vb = b.runs[j]
+		}
+		if i < na && (j >= nb || va.start < vb.start) {
+			output.runAppendInterval(va)
 			i++
 		} else {
-			if vb.start <= lastrun.last+1 {
-				if lastrun.last < vb.last {
-					lastrun.last = vb.last
-					output.runs[len(output.runs)-1] = lastrun
-				}
-			} else {
-				output.runs = append(output.runs, vb)
-			}
+			output.runAppendInterval(vb)
 			j++
 		}
 	}
-	var k int
-	var rest []interval32
-	if i < na {
-		k = i
-		rest = a.runs
-	} else {
-		k = j
-		rest = b.runs
-	}
-	for k < len(rest) {
-		vk := rest[k]
-		lastrun := output.runs[len(output.runs)-1]
-		if vk.start <= lastrun.last+1 {
-			if lastrun.last < vk.last {
-				lastrun.last = vk.last
-				output.runs[len(output.runs)-1] = lastrun
-			}
-		} else {
-			output.runs = append(output.runs, vk)
-		}
-		k++
-	}
 	return output
 }
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 0231bf93f..5c8e46f93 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -549,6 +549,31 @@ func TestUnionRunRun(t *testing.T) {
 			bruns: []interval32{{start: 5, last: 10}},
 			exp:   []interval32{{start: 1, last: 3}, {start: 5, last: 12}},
 		},
+		{
+			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			bruns: []interval32{{start: 2, last: 65535}},
+			exp:   []interval32{{start: 1, last: 65535}},
+		},
+		{
+			aruns: []interval32{{start: 2, last: 65535}},
+			bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			exp:   []interval32{{start: 1, last: 65535}},
+		},
+		{
+			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			bruns: []interval32{{start: 0, last: 65535}},
+			exp:   []interval32{{start: 0, last: 65535}},
+		},
+		{
+			aruns: []interval32{{start: 0, last: 65535}},
+			bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			exp:   []interval32{{start: 0, last: 65535}},
+		},
+		{
+			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}},
+			bruns: []interval32{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}},
+			exp:   []interval32{{start: 1, last: 9}, {start: 12, last: 27}, {start: 33, last: 34}},
+		},
 	}
 	for i, test := range tests {
 		a.runs = test.aruns
@@ -558,5 +583,4 @@ func TestUnionRunRun(t *testing.T) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
 		}
 	}
-
 }

From 97346baa94e69d5b61a204f735d52f36e05dd6e4 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 30 May 2017 15:57:32 -0500
Subject: [PATCH 017/100] unionArrayRun implementation

---
 roaring/roaring.go               | 28 +++++++++++++++++++---
 roaring/roaring_internal_test.go | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index c032a83cd..7e23fae82 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1860,8 +1860,31 @@ func unionArrayArray(a, b *container) *container {
 	return output
 }
 
+// unionArrayRun optimistically assumes that the result will be a run container,
+// and converts to a bitmap or array container afterwards if necessary.
 func unionArrayRun(a, b *container) *container {
-	return nil
+	output := &container{}
+	na, nb := len(a.array), len(b.runs)
+	var vb interval32
+	var va uint32
+	for i, j := 0, 0; i < na || j < nb; {
+		if i < na {
+			va = a.array[i]
+		}
+		if j < nb {
+			vb = b.runs[j]
+		}
+		if i < na && (j >= nb || va < vb.start) {
+			output.runAppendInterval(interval32{start: va, last: va})
+			i++
+		} else {
+			output.runAppendInterval(vb)
+			j++
+		}
+	}
+	return output
+}
+
 func (c *container) runAppendInterval(v interval32) {
 	if len(c.runs) == 0 {
 		c.runs = append(c.runs, v)
@@ -1880,9 +1903,8 @@ func unionRunRun(a, b *container) *container {
 	output := &container{
 		runs: make([]interval32, 0, na+nb),
 	}
-	i, j := 0, 0
 	var va, vb interval32
-	for i < na || j < nb {
+	for i, j := 0, 0; i < na || j < nb; {
 		if i < na {
 			va = a.runs[i]
 		}
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 5c8e46f93..324d4f670 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -584,3 +584,43 @@ func TestUnionRunRun(t *testing.T) {
 		}
 	}
 }
+
+func TestUnionArrayRun(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		array []uint32
+		runs  []interval32
+		exp   []interval32
+	}{
+		{
+			array: []uint32{1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 12}},
+		},
+		{
+			array: []uint32{},
+			runs:  []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 5, last: 10}},
+		},
+		{
+			array: []uint32{1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval32{},
+			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 5}, {start: 7, last: 7}, {start: 10, last: 12}},
+		},
+		{
+			array: []uint32{0, 1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval32{{start: 0, last: 5}, {start: 7, last: 7}},
+			exp:   []interval32{{start: 0, last: 5}, {start: 7, last: 7}, {start: 10, last: 12}},
+		},
+	}
+
+	for i, test := range tests {
+		a.array = test.array
+		b.runs = test.runs
+		ret := unionArrayRun(a, b)
+		if !reflect.DeepEqual(ret.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
+		}
+	}
+}

From 8ec8fc556c4b3acbfc67663ff855d5f8ffebf62d Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 30 May 2017 16:00:17 -0500
Subject: [PATCH 018/100] intersectRunRun to use runAppendInterval

this avoid situations where two runs which are adjacent don't get collapsed to a
single run
---
 roaring/roaring.go               | 7 ++++---
 roaring/roaring_internal_test.go | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 7e23fae82..e9ba462f1 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1684,19 +1684,20 @@ func intersectRunRun(a, b *container) *container {
 			j++
 		} else if va.last > vb.last && va.start >= vb.start {
 			// |--vb-|-|-va--|
-			output.runs = append(output.runs, interval32{start: va.start, last: vb.last})
+			output.runAppendInterval(interval32{start: va.start, last: vb.last})
 			j++
 		} else if va.last > vb.last && va.start < vb.start {
 			// |--va|--vb--|--|
-			output.runs = append(output.runs, vb)
+			output.runAppendInterval(vb)
 			j++
 		} else if va.last <= vb.last && va.start >= vb.start {
 			// |--vb|--va--|--|
 			output.runs = append(output.runs, va)
+			output.runAppendInterval(va)
 			i++
 		} else if va.last <= vb.last && va.start < vb.start {
 			// |--va-|-|-vb--|
-			output.runs = append(output.runs, interval32{start: vb.start, last: va.last})
+			output.runAppendInterval(interval32{start: vb.start, last: va.last})
 			i++
 		}
 	}
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 324d4f670..10458705c 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -413,7 +413,7 @@ func TestIntersectRunRun(t *testing.T) {
 		{
 			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
 			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 10}},
+			exp:   []interval32{{start: 5, last: 5}, {start: 7, last: 10}},
 		},
 	}
 	for i, test := range tests {

From c17c7372649b243524aad652648cae6af7dc08d6 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 30 May 2017 16:05:49 -0500
Subject: [PATCH 019/100] be smarter about setting output.n

---
 roaring/roaring.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index e9ba462f1..430c9c50c 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1659,10 +1659,10 @@ func intersectArrayRun(a, b *container) *container {
 			j++
 		} else {
 			output.array = append(output.array, va)
-			output.n++
 			i++
 		}
 	}
+	output.n = len(output.array)
 	return output
 }
 
@@ -1704,7 +1704,7 @@ func intersectRunRun(a, b *container) *container {
 	return output
 }
 
-// intersectBitmapRun returns an array container of the run container's
+// intersectBitmapRun returns an array container if the run container's
 // cardinality is < 4096. Otherwise it returns a bitmap container.
 func intersectBitmapRun(a, b *container) *container {
 	var output *container

From 88e0e4d5b62f533815d5f296c64d78c2f7602f20 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 30 May 2017 20:57:15 -0500
Subject: [PATCH 020/100] cleanup - setting cardinality and conversions

---
 roaring/roaring.go               | 59 ++++++++++++++++++++++++--------
 roaring/roaring_internal_test.go | 45 ++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 15 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 430c9c50c..ed3863dfc 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -892,6 +892,9 @@ func (itr *BufIterator) Unread() {
 // The maximum size of array containers.
 const ArrayMaxSize = 4096
 
+// The maximum size of run length encoded containers.
+const RunMaxSize = 2048
+
 // container represents a container for uint32 integers.
 //
 // These are used for storing the low bits. Containers are separated into two
@@ -1316,6 +1319,22 @@ func (c *container) arrayToBitmap() {
 	c.mapped = false
 }
 
+func (c *container) runToBitmap() {
+	// TODO
+}
+
+func (c *container) bitmapToRun() {
+	// TODO
+}
+
+func (c *container) arrayToRun() {
+	// TODO
+}
+
+func (c *container) runToArray() {
+	// TODO
+}
+
 // clone returns a copy of c.
 func (c *container) clone() *container {
 	other := &container{n: c.n}
@@ -1666,12 +1685,8 @@ func intersectArrayRun(a, b *container) *container {
 	return output
 }
 
-// intersectRunRun computes the intersect of two run containers. The output is
-// always a run container, since it can't possible have more runs than either of
-// the inputs. (note: it is possible that an array container would be better in
-// some cases, but probably not worth complicating the implementation)
+// intersectRunRun computes the intersect of two run containers.
 func intersectRunRun(a, b *container) *container {
-	// TODO need to take the last element of output.runs into account during each loop - possible that instead of appending, the last run should just be expanded
 	output := &container{}
 	na, nb := len(a.runs), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
@@ -1684,23 +1699,27 @@ func intersectRunRun(a, b *container) *container {
 			j++
 		} else if va.last > vb.last && va.start >= vb.start {
 			// |--vb-|-|-va--|
-			output.runAppendInterval(interval32{start: va.start, last: vb.last})
+			output.n += output.runAppendInterval(interval32{start: va.start, last: vb.last})
 			j++
 		} else if va.last > vb.last && va.start < vb.start {
 			// |--va|--vb--|--|
-			output.runAppendInterval(vb)
+			output.n += output.runAppendInterval(vb)
 			j++
 		} else if va.last <= vb.last && va.start >= vb.start {
 			// |--vb|--va--|--|
-			output.runs = append(output.runs, va)
-			output.runAppendInterval(va)
+			output.n += output.runAppendInterval(va)
 			i++
 		} else if va.last <= vb.last && va.start < vb.start {
 			// |--va-|-|-vb--|
-			output.runAppendInterval(interval32{start: vb.start, last: va.last})
+			output.n += output.runAppendInterval(interval32{start: vb.start, last: va.last})
 			i++
 		}
 	}
+	if n < 4096 && len(output.runs) > n/2 {
+		output.runToArray()
+	} else if len(output.runs) > RunMaxSize {
+		output.runToBitmap()
+	}
 	return output
 }
 
@@ -1886,17 +1905,27 @@ func unionArrayRun(a, b *container) *container {
 	return output
 }
 
-func (c *container) runAppendInterval(v interval32) {
+// runAppendInterval adds the given interval to the run container. It assumes
+// that the interval comes at the end of the list of runs, and does not check
+// that this is the case. It will not behave correctly if the start of the given
+// interval is earlier than the start of the last interval in the list of runs.
+// Its return value is the amount by which the cardinality of the container was
+// increased.
+func (c *container) runAppendInterval(v interval32) int {
 	if len(c.runs) == 0 {
 		c.runs = append(c.runs, v)
+		return int(v.last - v.start + 1)
 	} else {
-		lastidx := len(c.runs) - 1
-		if c.runs[lastidx].last+1 >= v.start && v.last > c.runs[lastidx].last {
-			c.runs[lastidx].last = v.last
-		} else if c.runs[lastidx].last+1 < v.start {
+		last := c.runs[len(c.runs)-1]
+		if last.last+1 >= v.start && v.last > last.last {
+			c.runs[len(c.runs)-1].last = v.last
+			return int(v.last - last.last)
+		} else if last.last+1 < v.start {
 			c.runs = append(c.runs, v)
+			return int(v.last - v.start + 1)
 		}
 	}
+	return 0
 }
 
 func unionRunRun(a, b *container) *container {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 10458705c..a106d3d8b 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -19,6 +19,44 @@ import (
 	"testing"
 )
 
+func TestRunAppendInterval(t *testing.T) {
+	a := container{}
+	tests := []struct {
+		base []interval32
+		app  interval32
+		exp  int
+	}{
+		{
+			base: []interval32{},
+			app:  interval32{start: 22, last: 25},
+			exp:  4,
+		},
+		{
+			base: []interval32{{start: 20, last: 23}},
+			app:  interval32{start: 22, last: 25},
+			exp:  2,
+		},
+		{
+			base: []interval32{{start: 20, last: 23}},
+			app:  interval32{start: 21, last: 22},
+			exp:  0,
+		},
+		{
+			base: []interval32{{start: 20, last: 23}},
+			app:  interval32{start: 19, last: 25},
+			exp:  3,
+		},
+	}
+
+	for i, test := range tests {
+		a.runs = test.base
+		if n := a.runAppendInterval(test.app); n != test.exp {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, n)
+		}
+	}
+
+}
+
 func TestInterval32RunLen(t *testing.T) {
 	iv := interval32{start: 7, last: 9}
 	if iv.runlen() != 3 {
@@ -399,21 +437,25 @@ func TestIntersectRunRun(t *testing.T) {
 		aruns []interval32
 		bruns []interval32
 		exp   []interval32
+		expN  int
 	}{
 		{
 			aruns: []interval32{},
 			bruns: []interval32{{start: 5, last: 10}},
 			exp:   []interval32(nil),
+			expN:  0,
 		},
 		{
 			aruns: []interval32{{start: 5, last: 12}},
 			bruns: []interval32{{start: 5, last: 10}},
 			exp:   []interval32{{start: 5, last: 10}},
+			expN:  6,
 		},
 		{
 			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
 			bruns: []interval32{{start: 5, last: 10}},
 			exp:   []interval32{{start: 5, last: 5}, {start: 7, last: 10}},
+			expN:  5,
 		},
 	}
 	for i, test := range tests {
@@ -423,6 +465,9 @@ func TestIntersectRunRun(t *testing.T) {
 		if !reflect.DeepEqual(ret.runs, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
 		}
+		if ret.n != test.expN {
+			t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n)
+		}
 	}
 
 }

From 487a369c6d4f5e2d0b35faeb0a8d2137781a197b Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 30 May 2017 21:57:57 -0500
Subject: [PATCH 021/100] more cardinality tracking and tests

---
 roaring/roaring.go               | 19 +++++++++++++++----
 roaring/roaring_internal_test.go | 21 +++++++++++++++++++++
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index ed3863dfc..76aa9a6b3 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1715,7 +1715,7 @@ func intersectRunRun(a, b *container) *container {
 			i++
 		}
 	}
-	if n < 4096 && len(output.runs) > n/2 {
+	if output.n < 4096 && len(output.runs) > output.n/2 {
 		output.runToArray()
 	} else if len(output.runs) > RunMaxSize {
 		output.runToBitmap()
@@ -1737,6 +1737,7 @@ func intersectBitmapRun(a, b *container) *container {
 				}
 			}
 		}
+		output.n = len(output.array)
 	} else {
 		// right now this iterates through the runs and sets integers in the
 		// bitmap that are in the runs. alternately, we could zero out ranges in
@@ -1752,15 +1753,22 @@ func intersectBitmapRun(a, b *container) *container {
 			for valast >= vb.start && vastart <= vb.last {
 				if vastart >= vb.start && valast <= vb.last { // a within b
 					output.bitmap[i] = a.bitmap[i]
+					output.n += int(popcnt(a.bitmap[i]))
 				} else if vb.start >= vastart && vb.last <= valast { // b within a
 					var mask uint64 = ((1 << (vb.last - vb.start + 1)) - 1) << (vb.start - vastart)
-					output.bitmap[i] |= a.bitmap[i] & mask
+					bits := a.bitmap[i] & mask
+					output.bitmap[i] |= bits
+					output.n += int(popcnt(bits))
 				} else if vastart < vb.start { // a overlaps front of b
 					offset := 64 - (1 + valast - vb.start)
-					output.bitmap[i] |= (a.bitmap[i] >> offset) << offset
+					bits := (a.bitmap[i] >> offset) << offset
+					output.bitmap[i] |= bits
+					output.n += int(popcnt(bits))
 				} else if vb.start < vastart { // b overlaps front of a
 					offset := 64 - (1 + vb.last - vastart)
-					output.bitmap[i] |= (a.bitmap[i] << offset) >> offset
+					bits := (a.bitmap[i] << offset) >> offset
+					output.bitmap[i] |= bits
+					output.n += int(popcnt(bits))
 				}
 				// update loop vars
 				i++
@@ -1768,6 +1776,9 @@ func intersectBitmapRun(a, b *container) *container {
 				valast = vastart + 63
 			}
 		}
+		if output.n < 4096 {
+			output.bitmapToArray()
+		}
 	}
 	return output
 }
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index a106d3d8b..efba7d85c 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -479,31 +479,37 @@ func TestIntersectBitmapRunBitmap(t *testing.T) {
 		bitmap []uint64
 		runs   []interval32
 		exp    []uint64
+		expN   int
 	}{
 		{
 			bitmap: []uint64{1},
 			runs:   []interval32{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}},
 			exp:    []uint64{1},
+			expN:   1,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
 			runs:   []interval32{{start: 1, last: 1}},
 			exp:    []uint64{2},
+			expN:   1,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
 			runs:   []interval32{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}},
 			exp:    []uint64{0xe000000000001C02},
+			expN:   7,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF},
 			runs:   []interval32{{start: 1, last: 1}, {start: 61, last: 77}},
 			exp:    []uint64{0xE000000000000002, 0x00000000000003FFF},
+			expN:   18,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1},
 			runs:   []interval32{{start: 63, last: 10000}},
 			exp:    []uint64{0x8000000000000000, 1, 1, 1, 0xA, 1, 1, 0, 1},
+			expN:   9,
 		},
 	}
 	for i, test := range tests {
@@ -517,9 +523,15 @@ func TestIntersectBitmapRunBitmap(t *testing.T) {
 			exp[i] = v
 		}
 		ret := intersectBitmapRun(a, b)
+		if ret.isArray() {
+			ret.arrayToBitmap()
+		}
 		if !reflect.DeepEqual(ret.bitmap, exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, exp, ret.bitmap)
 		}
+		if ret.n != test.expN {
+			t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n)
+		}
 	}
 
 }
@@ -531,31 +543,37 @@ func TestIntersectBitmapRunArray(t *testing.T) {
 		bitmap []uint64
 		runs   []interval32
 		exp    []uint32
+		expN   int
 	}{
 		{
 			bitmap: []uint64{1},
 			runs:   []interval32{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}},
 			exp:    []uint32{0},
+			expN:   1,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
 			runs:   []interval32{{start: 1, last: 1}},
 			exp:    []uint32{1},
+			expN:   1,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
 			runs:   []interval32{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}},
 			exp:    []uint32{1, 10, 11, 12, 61, 62, 63},
+			expN:   7,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF},
 			runs:   []interval32{{start: 1, last: 1}, {start: 61, last: 68}},
 			exp:    []uint32{1, 61, 62, 63, 64, 65, 66, 67, 68},
+			expN:   9,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1},
 			runs:   []interval32{{start: 63, last: 10000}},
 			exp:    []uint32{63, 64, 128, 192, 257, 259, 320, 384, 512},
+			expN:   9,
 		},
 	}
 	for i, test := range tests {
@@ -567,6 +585,9 @@ func TestIntersectBitmapRunArray(t *testing.T) {
 		if !reflect.DeepEqual(ret.array, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
 		}
+		if ret.n != test.expN {
+			t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n)
+		}
 	}
 
 }

From 32c3b2ad52c69b38ac5a70ec48d570f4d47aa5ba Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Tue, 30 May 2017 22:08:21 -0500
Subject: [PATCH 022/100] more cardinality and conversions

---
 roaring/roaring.go               | 25 +++++++++++++++++--------
 roaring/roaring_internal_test.go |  2 +-
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 76aa9a6b3..a10b5463c 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1715,7 +1715,7 @@ func intersectRunRun(a, b *container) *container {
 			i++
 		}
 	}
-	if output.n < 4096 && len(output.runs) > output.n/2 {
+	if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
 		output.runToArray()
 	} else if len(output.runs) > RunMaxSize {
 		output.runToBitmap()
@@ -1724,10 +1724,10 @@ func intersectRunRun(a, b *container) *container {
 }
 
 // intersectBitmapRun returns an array container if the run container's
-// cardinality is < 4096. Otherwise it returns a bitmap container.
+// cardinality is < ArrayMaxSize. Otherwise it returns a bitmap container.
 func intersectBitmapRun(a, b *container) *container {
 	var output *container
-	if b.n < 4096 {
+	if b.n < ArrayMaxSize {
 		// output is array container
 		output = &container{}
 		for _, iv := range b.runs {
@@ -1776,7 +1776,7 @@ func intersectBitmapRun(a, b *container) *container {
 				valast = vastart + 63
 			}
 		}
-		if output.n < 4096 {
+		if output.n < ArrayMaxSize {
 			output.bitmapToArray()
 		}
 	}
@@ -1906,13 +1906,18 @@ func unionArrayRun(a, b *container) *container {
 			vb = b.runs[j]
 		}
 		if i < na && (j >= nb || va < vb.start) {
-			output.runAppendInterval(interval32{start: va, last: va})
+			output.n += output.runAppendInterval(interval32{start: va, last: va})
 			i++
 		} else {
-			output.runAppendInterval(vb)
+			output.n += output.runAppendInterval(vb)
 			j++
 		}
 	}
+	if output.n < ArrayMaxSize {
+		output.runToArray()
+	} else if len(output.runs) > RunMaxSize {
+		output.runToBitmap()
+	}
 	return output
 }
 
@@ -1953,17 +1958,21 @@ func unionRunRun(a, b *container) *container {
 			vb = b.runs[j]
 		}
 		if i < na && (j >= nb || va.start < vb.start) {
-			output.runAppendInterval(va)
+			output.n += output.runAppendInterval(va)
 			i++
 		} else {
-			output.runAppendInterval(vb)
+			output.n += output.runAppendInterval(vb)
 			j++
 		}
 	}
+	if len(output.runs) > RunMaxSize {
+		output.runToBitmap()
+	}
 	return output
 }
 
 func unionBitmapRun(a, b *container) *container {
+	// TODO
 	return nil
 }
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index efba7d85c..7e1506ad2 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -44,7 +44,7 @@ func TestRunAppendInterval(t *testing.T) {
 		{
 			base: []interval32{{start: 20, last: 23}},
 			app:  interval32{start: 19, last: 25},
-			exp:  3,
+			exp:  2, // runAppendInterval explicitly does not support intervals whose start is < c.runs[-1].start
 		},
 	}
 

From 22393014cddde40053e8741dcbe5ad8655ba8baf Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 31 May 2017 13:52:17 -0500
Subject: [PATCH 023/100] Implement run/array conversion functions

---
 roaring/roaring.go | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index a10b5463c..ed1bdd317 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1327,12 +1327,33 @@ func (c *container) bitmapToRun() {
 	// TODO
 }
 
+// arrayToRun converts from array format to RLE format
 func (c *container) arrayToRun() {
-	// TODO
+	c.runs = make([]interval32, 0)
+	start := c.array[0]
+	for i, v := range c.array {
+		if v - c.array[i-1] > 1 {
+			// if current-previous > 1, one run ends and another begins
+			c.runs = append(c.runs, interval32{start, c.array[i-1]})
+			start = v
+		}
+	}
+	// append final run
+	c.runs = append(c.runs, interval32{start, c.array[i]})
+	c.array = nil
+	c.mapped = false
 }
 
+// runToArray converts from RLE format to array format
 func (c *container) runToArray() {
-	// TODO
+	c.array = make([]uint32, 0, c.n)
+	for _, r := range c.runs {
+		for v := r.start; v <= r.last; v++ {
+			c.array = append(c.array, v)
+		}
+	}
+	c.runs = nil
+	c.mapped = false
 }
 
 // clone returns a copy of c.

From 343e35f123bf9097d73189d69dd8632200689980 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 31 May 2017 14:30:32 -0500
Subject: [PATCH 024/100] Update TestUnionArrayRun for array result case

---
 roaring/roaring_internal_test.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 7e1506ad2..12417c80f 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -657,27 +657,27 @@ func TestUnionArrayRun(t *testing.T) {
 	tests := []struct {
 		array []uint32
 		runs  []interval32
-		exp   []interval32
+		exp   []uint32
 	}{
 		{
 			array: []uint32{1, 4, 5, 7, 10, 11, 12},
 			runs:  []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 12}},
+			exp:   []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12},
 		},
 		{
 			array: []uint32{},
 			runs:  []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 5, last: 10}},
+			exp:   []uint32{5, 6, 7, 8, 9, 10},
 		},
 		{
 			array: []uint32{1, 4, 5, 7, 10, 11, 12},
 			runs:  []interval32{},
-			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 5}, {start: 7, last: 7}, {start: 10, last: 12}},
+			exp:   []uint32{1, 4, 5, 7, 10, 11, 12},
 		},
 		{
 			array: []uint32{0, 1, 4, 5, 7, 10, 11, 12},
 			runs:  []interval32{{start: 0, last: 5}, {start: 7, last: 7}},
-			exp:   []interval32{{start: 0, last: 5}, {start: 7, last: 7}, {start: 10, last: 12}},
+			exp:   []uint32{0, 1, 2, 3, 4, 5, 7, 10, 11, 12},
 		},
 	}
 
@@ -685,7 +685,7 @@ func TestUnionArrayRun(t *testing.T) {
 		a.array = test.array
 		b.runs = test.runs
 		ret := unionArrayRun(a, b)
-		if !reflect.DeepEqual(ret.runs, test.exp) {
+		if !reflect.DeepEqual(ret.array, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
 		}
 	}

From 23ce2cd39489f343eff15400f870e60d74c5e9ae Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Wed, 31 May 2017 14:39:24 -0500
Subject: [PATCH 025/100] unionBitmapRun, bitmapSet/ZeroRange, special case
 optimization for RLE

---
 roaring/roaring.go               |  67 ++++++++++++++++-
 roaring/roaring_internal_test.go | 123 +++++++++++++++++++++++++++++++
 2 files changed, 188 insertions(+), 2 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index ed1bdd317..3dc643f2c 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1915,6 +1915,9 @@ func unionArrayArray(a, b *container) *container {
 // unionArrayRun optimistically assumes that the result will be a run container,
 // and converts to a bitmap or array container afterwards if necessary.
 func unionArrayRun(a, b *container) *container {
+	if b.n == 65536 {
+		return b.clone()
+	}
 	output := &container{}
 	na, nb := len(a.array), len(b.runs)
 	var vb interval32
@@ -1966,6 +1969,12 @@ func (c *container) runAppendInterval(v interval32) int {
 }
 
 func unionRunRun(a, b *container) *container {
+	if a.n == 65536 {
+		return a.clone()
+	}
+	if b.n == 65536 {
+		return b.clone()
+	}
 	na, nb := len(a.runs), len(b.runs)
 	output := &container{
 		runs: make([]interval32, 0, na+nb),
@@ -1993,8 +2002,62 @@ func unionRunRun(a, b *container) *container {
 }
 
 func unionBitmapRun(a, b *container) *container {
-	// TODO
-	return nil
+	if b.n == 65536 {
+		return b.clone()
+	}
+	output := a.clone()
+	for j := 0; j < len(b.runs); j++ {
+		output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last))
+	}
+	return output
+}
+
+const Z = 0xFFFFFFFFFFFFFFFF
+
+// sets all bits in [i, j] (inclusive) (c must be a bitmap container)
+func (c *container) bitmapSetRange(i, j uint64) {
+	j += 1
+	x := i / 64
+	y := (j - 1) / 64
+	var X uint64 = Z << (i % 64)
+	var Y uint64 = Z >> (64 - (j % 64))
+	xcnt := popcnt(X)
+	ycnt := popcnt(Y)
+	if x == y {
+		c.n += int((j - i) - popcnt(c.bitmap[x]&(X&Y)))
+		c.bitmap[x] |= (X & Y)
+	} else {
+		c.n += int(xcnt - popcnt(c.bitmap[x]&X))
+		c.bitmap[x] |= X
+		for i := x + 1; i < y; i++ {
+			c.n += int(64 - popcnt(c.bitmap[i]))
+			c.bitmap[i] = Z
+		}
+		c.n += int(ycnt - popcnt(c.bitmap[y]&Y))
+		c.bitmap[y] |= Y
+	}
+}
+
+// zeroes all bits in [i, j] (inclusive) (c must be a bitmap container)
+func (c *container) bitmapZeroRange(i, j uint64) {
+	j += 1
+	x := i / 64
+	y := (j - 1) / 64
+	var X uint64 = Z << (i % 64)
+	var Y uint64 = Z >> (64 - (j % 64))
+	if x == y {
+		c.n -= int(popcnt(c.bitmap[x] & (X & Y)))
+		c.bitmap[x] &= ^(X & Y)
+	} else {
+		c.n -= int(popcnt(c.bitmap[x] & X))
+		c.bitmap[x] &= ^X
+		for i := x + 1; i < y; i++ {
+			c.n -= int(popcnt(c.bitmap[i]))
+			c.bitmap[i] = 0
+		}
+		c.n -= int(popcnt(c.bitmap[y] & Y))
+		c.bitmap[y] &= ^Y
+	}
 }
 
 func unionArrayBitmap(a, b *container) *container {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 12417c80f..186e014bd 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -690,3 +690,126 @@ func TestUnionArrayRun(t *testing.T) {
 		}
 	}
 }
+
+func TestBitmapSetRange(t *testing.T) {
+	c := &container{bitmap: make([]uint64, bitmapN)}
+	tests := []struct {
+		bitmap []uint64
+		start  uint64
+		last   uint64
+		exp    []uint64
+		expN   int
+	}{
+		{
+			bitmap: []uint64{0x0000000000FFF900},
+			start:  9,
+			last:   10,
+			exp:    []uint64{0x0000000000FFFF00},
+			expN:   16,
+		},
+		{
+			bitmap: []uint64{0xFF0, 0xFF, 0xFF},
+			start:  60,
+			last:   130,
+			exp:    []uint64{0xF000000000000FF0, 0xFFFFFFFFFFFFFFFF, 0xFF},
+			expN:   84,
+		},
+	}
+
+	for i, test := range tests {
+		for i, v := range test.bitmap {
+			c.bitmap[i] = v
+		}
+		c.n = c.countRange(0, 65535)
+		c.bitmapSetRange(test.start, test.last)
+		if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) {
+			t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)])
+		}
+		if test.expN != c.n {
+			t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n)
+		}
+	}
+}
+
+func TestBitmapZeroRange(t *testing.T) {
+	c := &container{bitmap: make([]uint64, bitmapN)}
+	tests := []struct {
+		bitmap []uint64
+		start  uint64
+		last   uint64
+		exp    []uint64
+		expN   int
+	}{
+		{
+			bitmap: []uint64{0x0000000000FFFF00},
+			start:  9,
+			last:   10,
+			exp:    []uint64{0x0000000000FFF900},
+			expN:   14,
+		},
+		{
+			bitmap: []uint64{0xFF0, 0xFF, 0xFF},
+			start:  60,
+			last:   130,
+			exp:    []uint64{0xFF0, 0, 0xF8},
+			expN:   13,
+		},
+	}
+
+	for i, test := range tests {
+		for i, v := range test.bitmap {
+			c.bitmap[i] = v
+		}
+		c.n = c.countRange(0, 65535)
+		c.bitmapZeroRange(test.start, test.last)
+		if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) {
+			t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)])
+		}
+		if test.expN != c.n {
+			t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n)
+		}
+		for i, _ := range test.bitmap {
+			c.bitmap[i] = 0
+		}
+	}
+
+}
+
+func TestUnionBitmapRun(t *testing.T) {
+	a := &container{bitmap: make([]uint64, bitmapN)}
+	b := &container{}
+	tests := []struct {
+		bitmap []uint64
+		runs   []interval32
+		exp    []uint64
+		expN   int
+	}{
+		{
+			bitmap: []uint64{2},
+			runs:   []interval32{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 78}},
+			exp:    []uint64{0xC00000000000003F, 0x60FF},
+			expN:   18,
+		},
+	}
+	for i, test := range tests {
+		for i, v := range test.bitmap {
+			a.bitmap[i] = v
+		}
+		a.n = a.bitmapCountRange(0, 65535)
+		b.runs = test.runs
+		ret := unionBitmapRun(a, b)
+		if ret.isArray() {
+			ret.arrayToBitmap()
+		}
+		if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) {
+			t.Fatalf("test #%v expected %x, but got %x", i, test.exp, ret.bitmap[:len(test.exp)])
+		}
+		if ret.n != test.expN {
+			t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n)
+		}
+		for i, _ := range test.bitmap {
+			a.bitmap[i] = 0
+		}
+	}
+
+}

From dbee584866657c329ff5c163225662623228546a Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 31 May 2017 15:34:37 -0500
Subject: [PATCH 026/100] Fixes and tests for array-run conversions

---
 roaring/roaring.go               | 10 +++---
 roaring/roaring_internal_test.go | 61 +++++++++++++++++++++++++++++++-
 2 files changed, 65 insertions(+), 6 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 3dc643f2c..310ab78cb 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1329,17 +1329,17 @@ func (c *container) bitmapToRun() {
 
 // arrayToRun converts from array format to RLE format
 func (c *container) arrayToRun() {
-	c.runs = make([]interval32, 0)
+	c.runs = make([]interval32, 0, c.n) // what capacity to use?
 	start := c.array[0]
-	for i, v := range c.array {
-		if v - c.array[i-1] > 1 {
+	for i, v := range c.array[1:] {
+		if v - c.array[i] > 1 {
 			// if current-previous > 1, one run ends and another begins
-			c.runs = append(c.runs, interval32{start, c.array[i-1]})
+			c.runs = append(c.runs, interval32{start, c.array[i]})
 			start = v
 		}
 	}
 	// append final run
-	c.runs = append(c.runs, interval32{start, c.array[i]})
+	c.runs = append(c.runs, interval32{start, c.array[c.n-1]})
 	c.array = nil
 	c.mapped = false
 }
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 186e014bd..4424969ad 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -731,6 +731,36 @@ func TestBitmapSetRange(t *testing.T) {
 	}
 }
 
+func TestArrayToRun(t *testing.T) {
+	a := &container{}
+	tests := []struct {
+		array []uint32
+		exp   []interval32
+	}{
+		{
+			array: []uint32{0},
+			exp:   []interval32{{start: 0, last: 0}},
+		},
+		{
+			array: []uint32{0, 1, 2, 3, 4},
+			exp:   []interval32{{start: 0, last: 4}},
+		},
+		{
+			array: []uint32{2, 5, 6, 7, 13, 14, 17},
+			exp:   []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
+		},
+	}
+
+	for i, test := range tests {
+		a.array = test.array
+		a.n = len(test.array)
+		a.arrayToRun()
+		if !reflect.DeepEqual(a.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs)
+		}
+	}
+}
+
 func TestBitmapZeroRange(t *testing.T) {
 	c := &container{bitmap: make([]uint64, bitmapN)}
 	tests := []struct {
@@ -811,5 +841,34 @@ func TestUnionBitmapRun(t *testing.T) {
 			a.bitmap[i] = 0
 		}
 	}
-
+}
+
+func TestRunToArray(t *testing.T) {
+	a := &container{}
+	tests := []struct {
+		runs []interval32
+		exp  []uint32
+	}{
+		{
+			runs: []interval32{{start: 0, last: 0}},
+			exp:  []uint32{0},
+		},
+		{
+			runs: []interval32{{start: 0, last: 4}},
+			exp:  []uint32{0, 1, 2, 3, 4},
+		},
+		{
+			runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
+			exp:  []uint32{2, 5, 6, 7, 13, 14, 17},
+		},
+	}
+
+	for i, test := range tests {
+		a.runs = test.runs
+		a.n = len(test.exp)
+		a.runToArray()
+		if !reflect.DeepEqual(a.array, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array)
+		}
+	}
 }

From 1a8780c4a28ee09191f4bfaad9e71ac866c852b6 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Wed, 31 May 2017 21:10:06 -0500
Subject: [PATCH 027/100] add bitmapCountRuns and arrayCountRuns

---
 roaring/roaring.go               | 21 ++++++++
 roaring/roaring_internal_test.go | 91 ++++++++++++++++++++++++++++++++
 2 files changed, 112 insertions(+)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 310ab78cb..7c64472e3 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1164,6 +1164,27 @@ func (c *container) contains(v uint32) bool {
 	}
 }
 
+func (c *container) bitmapCountRuns() (r int) {
+	for i := 0; i < 1023; i++ {
+		v, v1 := c.bitmap[i], c.bitmap[i+1]
+		r = r + int(popcnt((v<<1)&^v)+((v>>63)&^v1))
+	}
+	vl := c.bitmap[len(c.bitmap)-1]
+	r = r + int(popcnt((vl<<1)&^vl)+vl>>63)
+	return r
+}
+
+func (c *container) arrayCountRuns() (r int) {
+	prev := -2
+	for _, v := range c.array {
+		if uint32(prev+1) != v {
+			r += 1
+		}
+		prev = int(v)
+	}
+	return r
+}
+
 func (c *container) arrayContains(v uint32) bool {
 	return search32(c.array, v) >= 0
 }
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 4424969ad..1d58c9b01 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -872,3 +872,94 @@ func TestRunToArray(t *testing.T) {
 		}
 	}
 }
+
+func TestBitmapCountRuns(t *testing.T) {
+	c := &container{bitmap: make([]uint64, bitmapN)}
+	tests := []struct {
+		bitmap []uint64
+		exp    int
+	}{
+		{
+			bitmap: []uint64{0xFF00FF00},
+			exp:    2,
+		},
+		{
+			bitmap: []uint64{0xFF00FF0000000000, 0x1},
+			exp:    2,
+		},
+		{
+			bitmap: []uint64{0xFF00FF0000000000, 0x2, 0x100},
+			exp:    4,
+		},
+		{
+			bitmap: []uint64{0xFF00FF0000000000, 0x1010101FF0101010, 0x100},
+			exp:    10,
+		},
+	}
+
+	for i, test := range tests {
+		for j, v := range test.bitmap {
+			c.bitmap[j] = v
+		}
+
+		ret := c.bitmapCountRuns()
+		if ret != test.exp {
+			t.Fatalf("test #%v expected %v but got %v", i, test.exp, ret)
+		}
+
+		for j, _ := range test.bitmap {
+			c.bitmap[j] = 0
+		}
+	}
+
+	test := tests[3]
+	for j, v := range test.bitmap {
+		c.bitmap[1024-len(test.bitmap)+j] = v
+
+	}
+	ret := c.bitmapCountRuns()
+	if ret != test.exp {
+		t.Fatalf("test at end expected %v but got %v", test.exp, ret)
+	}
+}
+
+func TestArrayCountRuns(t *testing.T) {
+	c := &container{}
+	tests := []struct {
+		array []uint32
+		exp   int
+	}{
+		{
+			array: []uint32{},
+			exp:   0,
+		},
+		{
+			array: []uint32{0},
+			exp:   1,
+		},
+		{
+			array: []uint32{1},
+			exp:   1,
+		},
+		{
+			array: []uint32{1, 2, 3, 5},
+			exp:   2,
+		},
+		{
+			array: []uint32{0, 1, 3, 9, 2048, 4096, 4097, 65534, 65535},
+			exp:   6,
+		},
+		{
+			array: []uint32{0, 10, 11, 12},
+			exp:   2,
+		},
+	}
+
+	for i, test := range tests {
+		c.array = test.array
+		ret := c.arrayCountRuns()
+		if ret != test.exp {
+			t.Fatalf("test #%v expected %v but got %v", i, test.exp, ret)
+		}
+	}
+}

From ac5f776db93dfbb7569cd2fb75423f3fdb6fd6f9 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Thu, 1 Jun 2017 08:31:16 -0500
Subject: [PATCH 028/100] optimize roaring on snapshot

---
 roaring/roaring.go | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 7c64472e3..f24e13fdd 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -503,8 +503,16 @@ func (b *Bitmap) countEmptyContainers() int {
 	return result
 }
 
+// Optimize converts array and bitmap containers to run containers as necessary.
+func (b *Bitmap) Optimize() {
+	for _, c := range b.containers {
+		c.Optimize()
+	}
+}
+
 // WriteTo writes b to w.
 func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
+	b.Optimize()
 	// Remove empty containers before persisting.
 	//b.removeEmptyContainers()
 	containerCount := len(b.keys) - b.countEmptyContainers()
@@ -1099,6 +1107,7 @@ func (c *container) arrayAdd(v uint32) bool {
 	copy(c.array[i+1:], c.array[i:])
 	c.array[i] = v
 	return true
+
 }
 
 func (c *container) bitmapAdd(v uint32) bool {
@@ -1185,6 +1194,20 @@ func (c *container) arrayCountRuns() (r int) {
 	return r
 }
 
+func (c *container) Optimize() {
+	if c.isArray() {
+		runs := c.arrayCountRuns()
+		if runs < c.n/2 {
+			c.arrayToRun()
+		}
+	} else if c.isBitmap() {
+		runs := c.bitmapCountRuns()
+		if runs < 2048 {
+			c.bitmapToRun()
+		}
+	}
+}
+
 func (c *container) arrayContains(v uint32) bool {
 	return search32(c.array, v) >= 0
 }

From bff12f5ce1025c6701809b6d37da6cfc205fb945 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 1 Jun 2017 10:54:22 -0500
Subject: [PATCH 029/100] Add implementations and minimal tests for bitmap-run
 conversions

---
 roaring/roaring.go               | 57 +++++++++++++++++++++++++-
 roaring/roaring_internal_test.go | 69 ++++++++++++++++++++++++++++++++
 2 files changed, 124 insertions(+), 2 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index f24e13fdd..bae6b60bb 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1363,12 +1363,65 @@ func (c *container) arrayToBitmap() {
 	c.mapped = false
 }
 
+// runToBitmap converts from RLE format to bitmap format
 func (c *container) runToBitmap() {
-	// TODO
+	c.bitmap = make([]uint64, bitmapN)
+	for _, r := range c.runs {
+		// TODO is there a faster way ?!?!!
+		for v := r.start; v <= r.last; v++ {
+			c.bitmap[int(v)/64] |= (uint64(1) << uint(v%64))
+		}
+	}
+	c.runs = nil
+	c.mapped = false
 }
 
+// bitmapToRun converts from bitmap format to RLE format
 func (c *container) bitmapToRun() {
-	// TODO
+	numRuns := c.n // TODO compute properly
+	c.runs = make([]interval32, 0, numRuns)
+	// TODO return early if no runs
+
+	current := c.bitmap[0]
+	var i, start, last uint32
+	for {
+		// skip while empty 
+		for current == 0 && i < bitmapN-1 {
+			i++
+			current = c.bitmap[i]
+		}
+
+		if current == 0 {
+			break
+		}
+		currentStart := uint32(trailingZeroN(current))
+		start = 64 * i + currentStart
+
+		// pad LSBs with 1s
+		current = current | (current-1)
+
+		// find next 0
+		for current == 0xFFFFFFFFFFFFFFFF && i < bitmapN-1 {
+			i++
+			current = c.bitmap[i]
+		}
+
+		last = 0
+		if current == 0xFFFFFFFFFFFFFFFF {
+			last = 64*i + 64  // TODO verify
+			c.runs = append(c.runs, interval32{start, last-1})
+			break
+		}
+		currentLast := uint32(trailingZeroN(^current))
+		last = 64 * i + currentLast
+		c.runs = append(c.runs, interval32{start, last-1})
+
+		// pad LSBs with 0s
+		current = current & (current+1)
+	}
+
+	c.bitmap = nil
+	c.mapped = false
 }
 
 // arrayToRun converts from array format to RLE format
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 1d58c9b01..e6643e5f5 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -731,6 +731,75 @@ func TestBitmapSetRange(t *testing.T) {
 	}
 }
 
+func TestRunToBitmap(t *testing.T) {
+	a := &container{}
+	tests := []struct {
+		runs []interval32
+		exp  []uint64
+	}{
+		{
+			runs: []interval32{{start: 0, last: 0}},
+			exp:  []uint64{1},
+		},
+		{
+			runs: []interval32{{start: 0, last: 4}},
+			exp:  []uint64{31},
+		},
+		{
+			runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
+			exp:  []uint64{155876},
+		},
+	}
+
+	for i, test := range tests {
+		exp := make([]uint64, bitmapN)
+		for i, v := range test.exp {
+			exp[i] = v
+		}
+
+		a.runs = test.runs
+		a.runToBitmap()
+		if !reflect.DeepEqual(a.bitmap, exp) {
+			t.Logf("test #%v expected %v, but got (%d) %v", i, exp, a.bitmap)
+		}
+	}
+}
+
+func TestBitmapToRun(t *testing.T) {
+	a := &container{}
+	tests := []struct {
+		bitmap []uint64
+		exp    []interval32
+	}{
+		{
+			bitmap: []uint64{1},
+			exp:    []interval32{{start: 0, last: 0}},
+		},
+		{
+			bitmap: []uint64{31},
+			exp:    []interval32{{start: 0, last: 4}},
+		},
+		{
+			bitmap: []uint64{155876},
+			exp:    []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
+		},
+	}
+
+	for i, test := range tests {
+		a.bitmap = make([]uint64, bitmapN)
+		n := 0
+		for i, v := range test.bitmap {
+			a.bitmap[i] = v
+			n += int(popcount(v))
+		}
+		a.n = n
+		a.bitmapToRun()
+		if !reflect.DeepEqual(a.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs)
+		}
+	}
+}
+
 func TestArrayToRun(t *testing.T) {
 	a := &container{}
 	tests := []struct {

From 60d38874745b62282d38079c35f9bb62939434de Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 1 Jun 2017 11:27:31 -0500
Subject: [PATCH 030/100] Update comments

---
 roaring/roaring.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index bae6b60bb..6d577eba3 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1339,7 +1339,7 @@ func (c *container) runMax() uint32 {
 	return uint32(c.runs[len(c.runs)-1].last)
 }
 
-// convertToArray converts the values in the bitmap to array values.
+// bitmapToArray converts from bitmap format to array format.
 func (c *container) bitmapToArray() {
 	c.array = make([]uint32, 0, c.n)
 	for i, bitmap := range c.bitmap {
@@ -1353,7 +1353,7 @@ func (c *container) bitmapToArray() {
 	c.mapped = false
 }
 
-// convertToBitmap converts the values in array to bitmap values.
+// arrayToBitmap converts from array format to bitmap format.
 func (c *container) arrayToBitmap() {
 	c.bitmap = make([]uint64, bitmapN)
 	for _, v := range c.array {
@@ -1363,7 +1363,7 @@ func (c *container) arrayToBitmap() {
 	c.mapped = false
 }
 
-// runToBitmap converts from RLE format to bitmap format
+// runToBitmap converts from RLE format to bitmap format.
 func (c *container) runToBitmap() {
 	c.bitmap = make([]uint64, bitmapN)
 	for _, r := range c.runs {
@@ -1376,7 +1376,7 @@ func (c *container) runToBitmap() {
 	c.mapped = false
 }
 
-// bitmapToRun converts from bitmap format to RLE format
+// bitmapToRun converts from bitmap format to RLE format.
 func (c *container) bitmapToRun() {
 	numRuns := c.n // TODO compute properly
 	c.runs = make([]interval32, 0, numRuns)
@@ -1424,7 +1424,7 @@ func (c *container) bitmapToRun() {
 	c.mapped = false
 }
 
-// arrayToRun converts from array format to RLE format
+// arrayToRun converts from array format to RLE format.
 func (c *container) arrayToRun() {
 	c.runs = make([]interval32, 0, c.n) // what capacity to use?
 	start := c.array[0]
@@ -1441,7 +1441,7 @@ func (c *container) arrayToRun() {
 	c.mapped = false
 }
 
-// runToArray converts from RLE format to array format
+// runToArray converts from RLE format to array format.
 func (c *container) runToArray() {
 	c.array = make([]uint32, 0, c.n)
 	for _, r := range c.runs {

From 8ee152cbadb9a325cad7e07b8a1db797e78e4e20 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 1 Jun 2017 13:16:49 -0500
Subject: [PATCH 031/100] Use meaningful name for maxBitmap const

---
 roaring/roaring.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 6d577eba3..702f4460d 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1401,13 +1401,13 @@ func (c *container) bitmapToRun() {
 		current = current | (current-1)
 
 		// find next 0
-		for current == 0xFFFFFFFFFFFFFFFF && i < bitmapN-1 {
+		for current == maxBitmap && i < bitmapN-1 {
 			i++
 			current = c.bitmap[i]
 		}
 
 		last = 0
-		if current == 0xFFFFFFFFFFFFFFFF {
+		if current == maxBitmap {
 			last = 64*i + 64  // TODO verify
 			c.runs = append(c.runs, interval32{start, last-1})
 			break
@@ -2109,15 +2109,15 @@ func unionBitmapRun(a, b *container) *container {
 	return output
 }
 
-const Z = 0xFFFFFFFFFFFFFFFF
+const maxBitmap = 0xFFFFFFFFFFFFFFFF
 
 // sets all bits in [i, j] (inclusive) (c must be a bitmap container)
 func (c *container) bitmapSetRange(i, j uint64) {
 	j += 1
 	x := i / 64
 	y := (j - 1) / 64
-	var X uint64 = Z << (i % 64)
-	var Y uint64 = Z >> (64 - (j % 64))
+	var X uint64 = maxBitmap << (i % 64)
+	var Y uint64 = maxBitmap >> (64 - (j % 64))
 	xcnt := popcnt(X)
 	ycnt := popcnt(Y)
 	if x == y {
@@ -2128,7 +2128,7 @@ func (c *container) bitmapSetRange(i, j uint64) {
 		c.bitmap[x] |= X
 		for i := x + 1; i < y; i++ {
 			c.n += int(64 - popcnt(c.bitmap[i]))
-			c.bitmap[i] = Z
+			c.bitmap[i] = maxBitmap
 		}
 		c.n += int(ycnt - popcnt(c.bitmap[y]&Y))
 		c.bitmap[y] |= Y
@@ -2140,8 +2140,8 @@ func (c *container) bitmapZeroRange(i, j uint64) {
 	j += 1
 	x := i / 64
 	y := (j - 1) / 64
-	var X uint64 = Z << (i % 64)
-	var Y uint64 = Z >> (64 - (j % 64))
+	var X uint64 = maxBitmap << (i % 64)
+	var Y uint64 = maxBitmap >> (64 - (j % 64))
 	if x == y {
 		c.n -= int(popcnt(c.bitmap[x] & (X & Y)))
 		c.bitmap[x] &= ^(X & Y)

From 2e540d275cab483f640341c37d2b9b1fb0d9112b Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 1 Jun 2017 14:21:46 -0500
Subject: [PATCH 032/100] Expand bitmap-run conversion tests

---
 roaring/roaring_internal_test.go | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index e6643e5f5..193423a61 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -749,6 +749,10 @@ func TestRunToBitmap(t *testing.T) {
 			runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
 			exp:  []uint64{155876},
 		},
+		{
+			runs: []interval32{{start: 0, last: 3}, {start: 60, last: 67}},
+			exp:  []uint64{0xF00000000000000F, 0x000000000000000F},
+		},
 	}
 
 	for i, test := range tests {
@@ -772,17 +776,40 @@ func TestBitmapToRun(t *testing.T) {
 		exp    []interval32
 	}{
 		{
+			// single-bit run
 			bitmap: []uint64{1},
 			exp:    []interval32{{start: 0, last: 0}},
 		},
 		{
+			// single multi-bit run in one word
 			bitmap: []uint64{31},
 			exp:    []interval32{{start: 0, last: 4}},
 		},
 		{
+			// multiple runs in one word
 			bitmap: []uint64{155876},
 			exp:    []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
 		},
+		{
+			// span two words, both mixed
+			bitmap: []uint64{0xF00000000000000F, 0x000000000000000F},
+			exp:    []interval32{{start: 0, last: 3}, {start: 60, last: 67}},
+		},
+		{
+			// span two words, first = maxBitmap
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xF},
+			exp:    []interval32{{start: 0, last: 67}},
+		},
+		{
+			// span two words, second = maxBitmap
+			bitmap: []uint64{0xF000000000000000, 0xFFFFFFFFFFFFFFFF},
+			exp:    []interval32{{start: 60, last: 127}},
+		},
+		{
+			// span three words
+			bitmap: []uint64{0xF000000000000000, 0xFFFFFFFFFFFFFFFF, 0xF},
+			exp:    []interval32{{start: 60, last: 131}},
+		},
 	}
 
 	for i, test := range tests {

From a51f7d66bb4ce7078fae559d3abd8c6c335ebd8a Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Mon, 5 Jun 2017 11:51:02 -0500
Subject: [PATCH 033/100] Add array-bitmap conversion test

---
 roaring/roaring_internal_test.go | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 193423a61..2504c0b36 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -731,6 +731,35 @@ func TestBitmapSetRange(t *testing.T) {
 	}
 }
 
+func TestArrayToBitmap(t *testing.T) {
+	a := &container{ctype: ARRAY}
+	tests := []struct {
+		array []uint32
+		exp   []uint64
+	}{
+		{
+			array: []uint32{0, 1, 2, 3},
+			exp:   []uint64{0xF},
+		},
+	}
+
+	for i, test := range tests {
+		exp := make([]uint64, bitmapN)
+		for i, v := range test.exp {
+			exp[i] = v
+		}
+
+		a.array = test.array
+		a.arrayToBitmap()
+		if !reflect.DeepEqual(a.bitmap, exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap)
+		}
+		if a.ctype != BITMAP {
+			t.Fatalf("test #%v expected type %v but got %v", i, BITMAP, a.ctype)
+		}
+	}
+}
+
 func TestRunToBitmap(t *testing.T) {
 	a := &container{}
 	tests := []struct {
@@ -764,7 +793,7 @@ func TestRunToBitmap(t *testing.T) {
 		a.runs = test.runs
 		a.runToBitmap()
 		if !reflect.DeepEqual(a.bitmap, exp) {
-			t.Logf("test #%v expected %v, but got (%d) %v", i, exp, a.bitmap)
+			t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap)
 		}
 	}
 }

From 3ad1c5b331a9d018f5eaf8183c0a4de13074e0dd Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Mon, 5 Jun 2017 11:57:24 -0500
Subject: [PATCH 034/100] Fix broken test

---
 roaring/roaring_internal_test.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 2504c0b36..89c85e6cc 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -732,7 +732,7 @@ func TestBitmapSetRange(t *testing.T) {
 }
 
 func TestArrayToBitmap(t *testing.T) {
-	a := &container{ctype: ARRAY}
+	a := &container{}
 	tests := []struct {
 		array []uint32
 		exp   []uint64
@@ -754,9 +754,6 @@ func TestArrayToBitmap(t *testing.T) {
 		if !reflect.DeepEqual(a.bitmap, exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap)
 		}
-		if a.ctype != BITMAP {
-			t.Fatalf("test #%v expected type %v but got %v", i, BITMAP, a.ctype)
-		}
 	}
 }
 

From 7e9fc3953d95e45cb063a1c222a6d04d72c32b51 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Tue, 6 Jun 2017 15:13:48 -0500
Subject: [PATCH 035/100] Support RLE in writing and reading snapshots

---
 roaring/roaring.go               | 100 +++++++++++++++++++++++++------
 roaring/roaring_internal_test.go |  41 +++++++++++++
 2 files changed, 124 insertions(+), 17 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 702f4460d..6ef1e17d3 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -27,10 +27,12 @@ import (
 
 const (
 	// cookie is the first four bytes in a roaring bitmap file.
-	cookie = uint32(12346)
+	cookieNoRuns = uint32(12346)
+	cookie = uint32(12347)
 
-	// headerSize is the size of the cookie and key count at the beginning of a file.
-	headerSize = 4 + 4
+	// headerBaseSize is the size of the cookie and key count at the beginning of a file.
+	// Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8.
+	headerBaseSize = 4 + 4
 
 	// bitmapN is the number of values in a container.bitmap.
 	bitmapN = (1 << 16) / 64
@@ -517,10 +519,41 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 	//b.removeEmptyContainers()
 	containerCount := len(b.keys) - b.countEmptyContainers()
 
+	// Create bitset indicating runs, record whether any runs present.
+	containsRuns := false
+	runFlagBitset := make([]uint8, (containerCount+7)/8)  // TODO verify size
+	var k uint8 = 0
+	for _, c := range b.containers {
+		if c.n == 0 {
+			continue
+		}
+		if c.isRun() {
+			containsRuns = true
+			runFlagBitset[k/8] |= (1 << (k%8))  // TODO verify
+		}
+		k++
+	}
+
+	thisCookie := cookieNoRuns
+	headerSize := headerBaseSize
+	if containsRuns {
+		thisCookie = cookie
+		headerSize += len(runFlagBitset)
+	}
+
 	// Build header before writing individual container blocks.
 	buf := make([]byte, headerSize+(containerCount*(4+8+4)))
-	binary.LittleEndian.PutUint32(buf[0:], cookie)
+	binary.LittleEndian.PutUint32(buf[0:], thisCookie)
+	// TODO supposed to be 2 bytes, but doesnt make sense for >65535 containers
 	binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount))
+
+	if containsRuns {
+		// Write runFlag bitset.
+		for i, b := range runFlagBitset {
+			buf[8+i] = b
+		}
+	}
+
 	empty := 0
 	// Encode keys and cardinality.
 	for i, key := range b.keys {
@@ -538,6 +571,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 		}
 	}
 
+	// TODO this should now be done conditionally
 	// Write the offset for each container block.
 	offset := uint32(len(buf))
 	empty = 0
@@ -574,12 +608,18 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 
 // UnmarshalBinary decodes b from a binary-encoded byte slice.
 func (b *Bitmap) UnmarshalBinary(data []byte) error {
-	if len(data) < headerSize {
+	if len(data) < headerBaseSize {
 		return errors.New("data too small")
 	}
 
-	// Verify the first 4 bytes are the correct cookie.
-	if v := binary.LittleEndian.Uint32(data[0:4]); v != cookie {
+	// Verify the first 4 bytes are a valid cookie.
+	v := binary.LittleEndian.Uint32(data[0:4])
+	containsRuns := false
+	if v == cookieNoRuns {
+		// noop
+	} else if v == cookie {
+		containsRuns = true
+	} else {
 		return errors.New("invalid roaring file")
 	}
 
@@ -588,17 +628,28 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 	b.keys = make([]uint64, keyN)
 	b.containers = make([]*container, keyN)
 
+	headerSize := headerBaseSize
+
+	runFlagBitset := make([]uint8, (keyN+7)/8)  // TODO verify size
+	if containsRuns {
+		// Read runFlag bitset.
+		for i := 0; i<len(runFlagBitset); i++ {
+			runFlagBitset[i] = data[8+i]
+		}
+		headerSize += len(runFlagBitset)
+	}
+
 	// Read container key headers.
-	for i, buf := 0, data[8:]; i < int(keyN); i, buf = i+1, buf[12:] {
+	for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
 		b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
 		b.containers[i] = &container{
 			n:      int(binary.LittleEndian.Uint32(buf[8:12])) + 1,
 			mapped: true,
 		}
 	}
+	opsOffset := headerSize + int(keyN)*12
 
 	// Read container offsets and attach data.
-	opsOffset := 8 + int(keyN)*12
 	for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] {
 		offset := binary.LittleEndian.Uint32(buf[0:4])
 
@@ -610,13 +661,22 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		// Map byte slice directly to the container data.
 		c := b.containers[i]
 		if c.n <= ArrayMaxSize {
-			c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
-			// TODO: instead of commenting this out, we need to make it a configuration option
-			//for _, v := range c.array {
-			//    assert(lowbits(uint64(v)) == v, "array value out of range: %d", v)
-			//}
-			opsOffset = int(offset) + len(c.array)*4
+			if containsRuns && (runFlagBitset[uint8(i)/8] & (1 << uint8(i)%8)) != 0 {
+				// Read runs.
+				runCount := binary.LittleEndian.Uint16(data[offset:offset+2])
+				c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+2]))[:runCount] // TODO verify
+				opsOffset = int(offset) + 2 + len(c.runs)*8 // TODO verify
+			} else {
+				// Read array.
+				c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
+				// TODO: instead of commenting this out, we ne ed to make it a configuration option
+				//for _, v := range c.array {
+				//    assert(lowbits(uint64(v)) == v, "array value out of range: %d", v)
+				//}
+				opsOffset = int(offset) + len(c.array)*4
+			}
 		} else {
+			// Read bitmap.
 			c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
 			opsOffset = int(offset) + len(c.bitmap)*8
 		}
@@ -1507,8 +1567,14 @@ func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) {
 }
 
 func (c *container) runWriteTo(w io.Writer) (n int64, err error) {
-	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:8*c.n])
-	return int64(nn), err
+	// TODO according to spec this should be [16-bit-runcount, start0, len0, start1, len1, ...]
+	// not sure what difference len vs last makes
+	err = binary.Write(w, binary.LittleEndian, uint16(len(c.runs)))
+	if err != nil {
+		return 0, err
+	}
+	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:8*len(c.runs)])
+	return int64(2+nn), err
 }
 
 // size returns the encoded size of the container, in bytes.
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 89c85e6cc..d76282a32 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -15,6 +15,7 @@
 package roaring
 
 import (
+	"bytes"
 	"reflect"
 	"testing"
 )
@@ -1085,3 +1086,43 @@ func TestArrayCountRuns(t *testing.T) {
 		}
 	}
 }
+
+func TestWriteReadArray(t *testing.T) {
+	ca := &container{array: []uint32{1, 10, 100, 1000}, n: 4}
+	ba := &Bitmap{keys: []uint64{0}, containers: []*container{ca}}
+	ba2 := &Bitmap{}
+	var buf bytes.Buffer
+	ba.WriteTo(&buf)
+	ba2.UnmarshalBinary(buf.Bytes())
+	if !reflect.DeepEqual(ba2.containers[0].array, ca.array) {
+		t.Fatalf("array test expected %x, but got %x", ca.array, ba2.containers[0].array)
+	}
+}
+
+func TestWriteReadBitmap(t *testing.T) {
+	// create bitmap containing > 4096 bits
+	cb := &container{bitmap: make([]uint64, bitmapN), n: 129*32}
+	for i := 0; i < 129 ; i++ {
+		cb.bitmap[i] = 0x5555555555555555
+	}
+	bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}}
+	bb2 := &Bitmap{}
+	var buf bytes.Buffer
+	bb.WriteTo(&buf)
+	bb2.UnmarshalBinary(buf.Bytes())
+	if !reflect.DeepEqual(bb2.containers[0].bitmap, cb.bitmap) {
+		t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.containers[0].bitmap)
+	}
+}
+
+func TestWriteReadRun(t *testing.T) {
+	cr := &container{runs: []interval32{{start: 3, last: 13}, {start: 100, last:109}}, n: 20}
+	br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}}
+	br2 := &Bitmap{}
+	var buf bytes.Buffer
+	br.WriteTo(&buf)
+	br2.UnmarshalBinary(buf.Bytes())
+	if !reflect.DeepEqual(br2.containers[0].runs, cr.runs) {
+		t.Fatalf("run test expected %x, but got %x", cr.runs, br2.containers[0].runs)
+	}
+}

From ff9d97e4bec85170518103997731ced107362931 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Fri, 9 Jun 2017 11:57:12 -0500
Subject: [PATCH 036/100] Implement differenceRunRun and add difference test
 shells

---
 roaring/roaring.go               |  80 +++++++++++++++++--
 roaring/roaring_internal_test.go | 133 +++++++++++++++++++++++++++++++
 2 files changed, 206 insertions(+), 7 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 6ef1e17d3..f5945a8e9 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2321,24 +2321,90 @@ func differenceArrayArray(a, b *container) *container {
 }
 
 func differenceArrayRun(a, b *container) *container {
-	// TODO
-	return nil
+	// func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
+
+	output := &container{array: make([]uint32, 0, a.n)}
+	// cardinality upper bound: card(A)
+	
+	return output
 }
 func differenceRunArray(a, b *container) *container {
 	// TODO
-	return nil
+	output := &container{runs: make([]interval32, 0, a.n)}
+	return output
 }
 func differenceRunBitmap(a, b *container) *container {
 	// TODO
-	return nil
+	output := &container{runs: make([]interval32, 0, a.n)}
+	return output
 }
 func differenceBitmapRun(a, b *container) *container {
 	// TODO
-	return nil
+	output := &container{bitmap: make([]uint64, bitmapN)}
+	return output
 }
 func differenceRunRun(a, b *container) *container {
-	// TODO
-	return nil
+	// (rc *runContainer32) AndNotRunContainer32(b *runContainer32) *runContainer32 {
+
+	if a.n == 0 || b.n == 0 {
+		return a
+	}
+
+	apos := 0  // current a-run index
+	bpos := 0  // current b-run index
+	astart := a.runs[apos].start
+	alast := a.runs[apos].last
+	bstart := b.runs[bpos].start
+	blast := b.runs[bpos].last
+	alen := len(a.runs)
+	blen := len(b.runs)
+
+	output := &container{runs: make([]interval32, 0, alen+blen)}  // TODO allocate max then truncate? or something else
+	// cardinality upper bound: sum of number of runs
+	// each B-run could split an A-run in two, up to len(b.runs) times
+
+	for apos < alen && bpos < blen {
+		switch {
+		case alast < bstart:
+			// current A-run entirely preceeds current B-run: keep full A-run, advance to next A-run
+			output.runs = append(output.runs, interval32{start: uint32(astart), last: uint32(alast)})
+			apos++
+			if apos < alen {
+				astart = a.runs[apos].start
+				alast = a.runs[apos].last
+			}
+		case blast < astart:
+			// current B-run entirely preceeds current A-run: advance to next B-run
+			bpos++
+			if bpos < blen {
+				bstart = b.runs[bpos].start
+				blast = b.runs[bpos].last
+			}
+		default:
+			// overlap
+			if astart < bstart {
+				output.runs = append(output.runs, interval32{start: uint32(astart), last: uint32(bstart - 1)})
+			}
+			if alast > blast {
+				astart = blast + 1
+			} else {
+				apos++
+				if apos < alen {
+					astart = a.runs[apos].start
+					alast = a.runs[apos].last
+				}
+			}
+		}
+	}
+	if apos < alen {
+		output.runs = append(output.runs, interval32{start: uint32(astart), last: uint32(alast)})
+		apos++
+		if apos < alen {
+			output.runs = append(output.runs, a.runs[apos:]...)
+		}
+	}
+
+	return output
 }
 
 func differenceArrayBitmap(a, b *container) *container {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index d76282a32..be2b07590 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1087,6 +1087,139 @@ func TestArrayCountRuns(t *testing.T) {
 	}
 }
 
+func TestDifferenceArrayRun(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		array []uint32
+		runs  []interval32
+		exp   []uint32
+	}{
+		{
+			array: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+			runs:  []interval32{{start: 5, last: 10}},
+			exp:   []uint32{0, 1, 2, 3, 4, 11, 12},
+		},
+	}
+	for i, test := range tests {
+		a.array = test.array
+		b.runs = test.runs
+		b.n = b.runCountRange(0, 100)
+		ret := differenceArrayRun(a, b)
+		if !reflect.DeepEqual(ret.array, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
+		}
+	}
+}
+
+func TestDifferenceRunArray(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		runs  []interval32
+		array []uint32
+		exp   []interval32
+	}{
+		{
+			runs:  []interval32{{start: 0, last: 12}},
+			array: []uint32{5, 6, 7, 8, 9, 10},
+			exp:   []interval32{{start: 0, last: 4}, {start: 11, last: 12}},
+		},
+	}
+	for i, test := range tests {
+		a.runs = test.runs
+		a.n = a.runCountRange(0, 100)
+		b.array = test.array
+		ret := differenceRunArray(a, b)
+		if !reflect.DeepEqual(ret.array, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
+		}
+	}
+}
+
+func TestDifferenceRunBitmap(t *testing.T) {
+	a := &container{}
+	b := &container{bitmap: make([]uint64, bitmapN)}
+	tests := []struct {
+		runs   []interval32
+		bitmap []uint64
+		exp    []interval32
+	}{
+		{
+			runs:   []interval32{{start: 0, last: 63}},
+			bitmap: []uint64{0x0000FFFF000000F0},
+			exp:    []interval32{{start: 0, last: 3}, {start: 8, last: 31}, {start: 48, last: 63}},
+		},
+	}
+	for i, test := range tests {
+		a.runs = test.runs
+		a.n = a.runCountRange(0, 100)
+		for i, v := range test.bitmap {
+			b.bitmap[i] = v
+		}
+		ret := differenceRunBitmap(a, b)
+		if !reflect.DeepEqual(ret.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
+		}
+	}
+}
+
+func TestDifferenceBitmapRun(t *testing.T) {
+	a := &container{bitmap: make([]uint64, bitmapN)}
+	b := &container{}
+	tests := []struct {
+		bitmap []uint64
+		runs   []interval32
+		exp    []uint64
+	}{
+		{
+			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
+			runs:   []interval32{{start: 4, last: 7}, {start: 32, last: 47}},
+			exp:    []uint64{0xFFFF0000FFFFFF0F},
+		},
+	}
+	for i, test := range tests {
+		for i, v := range test.bitmap {
+			a.bitmap[i] = v
+		}
+		b.runs = test.runs
+		b.n = b.runCountRange(0, 100)
+		ret := differenceBitmapRun(a, b)
+		if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap[:len(test.exp)])
+		}
+	}
+}
+
+func TestDifferenceRunRun(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		aruns []interval32
+		bruns []interval32
+		exp   []interval32
+	}{
+		{
+			// this tests all six overlap combinations
+			// A                     [   ]                   [   ]                        [ ]              [     ]             [   ]                 [  ]
+			// B                    [     ]                [   ]                     [ ]                     [  ]                [   ]                    [  ]
+			aruns: []interval32{{start: 3, last: 6}, {start: 13, last: 16}, {start: 24, last: 26}, {start: 33, last: 38}, {start: 43, last: 46}, {start: 53, last: 56}},
+			bruns: []interval32{{start: 1, last: 8}, {start: 11, last: 14}, {start: 21, last: 23}, {start: 35, last: 37}, {start: 44, last: 48}, {start: 57, last: 59}},
+			exp:   []interval32{{start: 15, last: 16}, {start: 24, last: 26}, {start: 33, last: 34}, {start: 38, last: 38}, {start: 43, last: 43}, {start: 53, last: 56}},
+		},
+	}
+	for i, test := range tests {
+		a.runs = test.aruns
+		a.n = a.runCountRange(0, 100)
+		b.runs = test.bruns
+		b.n = b.runCountRange(0, 100)
+		ret := differenceRunRun(a, b)
+		if !reflect.DeepEqual(ret.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
+		}
+	}
+}
+
 func TestWriteReadArray(t *testing.T) {
 	ca := &container{array: []uint32{1, 10, 100, 1000}, n: 4}
 	ba := &Bitmap{keys: []uint64{0}, containers: []*container{ca}}

From f6d3efe29d43cfaf40218b5acd53495306e3cb51 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Fri, 9 Jun 2017 13:45:06 -0500
Subject: [PATCH 037/100] Implement differenceBitmapRun and differenceArrayRun

---
 roaring/roaring.go | 67 +++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 60 insertions(+), 7 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index f5945a8e9..92f9e7c2e 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2323,31 +2323,84 @@ func differenceArrayArray(a, b *container) *container {
 func differenceArrayRun(a, b *container) *container {
 	// func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
 
+	if b.n == 0 {
+		return a.clone()
+	}
+
 	output := &container{array: make([]uint32, 0, a.n)}
 	// cardinality upper bound: card(A)
+
+	i := 0  // array index
+	j := 0  // run index
+
+	// keep all array elements before beginning of runs
+	for ; i < b.runs[j].start; i++ {
+		output.array = append(output.array, a.array[i])
+	}
+
+	// handle overlap
+	for ; i < a.n; i++ {
+		if !(a.array[i] >= b.runs[j].start && a.array[i] <= b.runs[j].last) {
+			output.array = append(output.array, a.array[i])
+		}
+		if i >= int(b.runs[j].last) {
+			j++
+			if j == len(b.runs) {
+				break
+			}
+		}
+	}
+	i++
+
+	// keep all array elements after end of runs
+	output.array = append(output.array, a.array[i:]...)
 	
 	return output
 }
+
+func differenceBitmapRun(a, b *container) *container {
+	if a.n == 0 || b.n == 0 {
+		return a.clone()
+	}
+
+	output := a.clone()
+	for j := 0; j < len(b.runs); j++ {
+		output.bitmapZeroRange(uint64(b.runs[j].start), uint64(b.runs[j].last))
+	}
+	return output
+}
+
 func differenceRunArray(a, b *container) *container {
 	// TODO
 	output := &container{runs: make([]interval32, 0, a.n)}
 	return output
 }
+
 func differenceRunBitmap(a, b *container) *container {
-	// TODO
 	output := &container{runs: make([]interval32, 0, a.n)}
+	itr := newBufIterator(newBitmapIterator(b.bitmap))
+
+	fmt.Printf("\ndifferenceRunBitmap\n")
+	for i := 0; ; {
+		vb, eof := itr.next()
+		if eof || {
+			break
+		}
+
+		fmt.Println(i, vb, eof)
+
+		i++
+	}
+
+
 	return output
 }
-func differenceBitmapRun(a, b *container) *container {
-	// TODO
-	output := &container{bitmap: make([]uint64, bitmapN)}
-	return output
-}
+
 func differenceRunRun(a, b *container) *container {
 	// (rc *runContainer32) AndNotRunContainer32(b *runContainer32) *runContainer32 {
 
 	if a.n == 0 || b.n == 0 {
-		return a
+		return a.clone()
 	}
 
 	apos := 0  // current a-run index

From bd90254282a3d875e2122e6c4a8667262d916435 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 9 Jun 2017 17:47:22 -0500
Subject: [PATCH 038/100] xorArrayRun;xorRunRun of rle

---
 roaring/roaring.go               | 215 ++++++++++++++++++++++++++++---
 roaring/roaring_internal_test.go | 106 ++++++++++++++-
 2 files changed, 300 insertions(+), 21 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 92f9e7c2e..2da7511b1 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -28,7 +28,7 @@ import (
 const (
 	// cookie is the first four bytes in a roaring bitmap file.
 	cookieNoRuns = uint32(12346)
-	cookie = uint32(12347)
+	cookie       = uint32(12347)
 
 	// headerBaseSize is the size of the cookie and key count at the beginning of a file.
 	// Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8.
@@ -521,7 +521,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 
 	// Create bitset indicating runs, record whether any runs present.
 	containsRuns := false
-	runFlagBitset := make([]uint8, (containerCount+7)/8)  // TODO verify size
+	runFlagBitset := make([]uint8, (containerCount+7)/8) // TODO verify size
 	var k uint8 = 0
 	for _, c := range b.containers {
 		if c.n == 0 {
@@ -529,7 +529,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 		}
 		if c.isRun() {
 			containsRuns = true
-			runFlagBitset[k/8] |= (1 << (k%8))  // TODO verify
+			runFlagBitset[k/8] |= (1 << (k % 8)) // TODO verify
 		}
 		k++
 	}
@@ -630,10 +630,10 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 
 	headerSize := headerBaseSize
 
-	runFlagBitset := make([]uint8, (keyN+7)/8)  // TODO verify size
+	runFlagBitset := make([]uint8, (keyN+7)/8) // TODO verify size
 	if containsRuns {
 		// Read runFlag bitset.
-		for i := 0; i<len(runFlagBitset); i++ {
+		for i := 0; i < len(runFlagBitset); i++ {
 			runFlagBitset[i] = data[8+i]
 		}
 		headerSize += len(runFlagBitset)
@@ -661,11 +661,11 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		// Map byte slice directly to the container data.
 		c := b.containers[i]
 		if c.n <= ArrayMaxSize {
-			if containsRuns && (runFlagBitset[uint8(i)/8] & (1 << uint8(i)%8)) != 0 {
+			if containsRuns && (runFlagBitset[uint8(i)/8]&(1<<uint8(i)%8)) != 0 {
 				// Read runs.
-				runCount := binary.LittleEndian.Uint16(data[offset:offset+2])
+				runCount := binary.LittleEndian.Uint16(data[offset : offset+2])
 				c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+2]))[:runCount] // TODO verify
-				opsOffset = int(offset) + 2 + len(c.runs)*8 // TODO verify
+				opsOffset = int(offset) + 2 + len(c.runs)*8                                   // TODO verify
 			} else {
 				// Read array.
 				c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
@@ -1445,7 +1445,7 @@ func (c *container) bitmapToRun() {
 	current := c.bitmap[0]
 	var i, start, last uint32
 	for {
-		// skip while empty 
+		// skip while empty
 		for current == 0 && i < bitmapN-1 {
 			i++
 			current = c.bitmap[i]
@@ -1455,10 +1455,10 @@ func (c *container) bitmapToRun() {
 			break
 		}
 		currentStart := uint32(trailingZeroN(current))
-		start = 64 * i + currentStart
+		start = 64*i + currentStart
 
 		// pad LSBs with 1s
-		current = current | (current-1)
+		current = current | (current - 1)
 
 		// find next 0
 		for current == maxBitmap && i < bitmapN-1 {
@@ -1468,16 +1468,16 @@ func (c *container) bitmapToRun() {
 
 		last = 0
 		if current == maxBitmap {
-			last = 64*i + 64  // TODO verify
-			c.runs = append(c.runs, interval32{start, last-1})
+			last = 64*i + 64 // TODO verify
+			c.runs = append(c.runs, interval32{start, last - 1})
 			break
 		}
 		currentLast := uint32(trailingZeroN(^current))
-		last = 64 * i + currentLast
-		c.runs = append(c.runs, interval32{start, last-1})
+		last = 64*i + currentLast
+		c.runs = append(c.runs, interval32{start, last - 1})
 
 		// pad LSBs with 0s
-		current = current & (current+1)
+		current = current & (current + 1)
 	}
 
 	c.bitmap = nil
@@ -1489,7 +1489,7 @@ func (c *container) arrayToRun() {
 	c.runs = make([]interval32, 0, c.n) // what capacity to use?
 	start := c.array[0]
 	for i, v := range c.array[1:] {
-		if v - c.array[i] > 1 {
+		if v-c.array[i] > 1 {
 			// if current-previous > 1, one run ends and another begins
 			c.runs = append(c.runs, interval32{start, c.array[i]})
 			start = v
@@ -1574,7 +1574,7 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) {
 		return 0, err
 	}
 	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:8*len(c.runs)])
-	return int64(2+nn), err
+	return int64(2 + nn), err
 }
 
 // size returns the encoded size of the container, in bytes.
@@ -2910,3 +2910,182 @@ func assert(condition bool, format string, a ...interface{}) {
 		panic(fmt.Sprintf(format, a...))
 	}
 }
+
+func xorArrayRun(a, b *container) *container {
+	output := &container{}
+	na, nb := len(a.array), len(b.runs)
+	var vb interval32
+	var va uint32
+	last_i, last_j := -1, -1
+	for i, j := 0, 0; i < na || j < nb; {
+		if i < na && i != last_i {
+			va = a.array[i]
+		}
+		if j < nb && j != last_j {
+			vb = b.runs[j]
+		}
+		last_i = i
+		last_j = j
+
+		if i < na && (j >= nb || va < vb.start) { //before
+			output.n += output.runAppendInterval(interval32{start: va, last: va})
+			i++
+		} else if j < nb && (i >= na || va > vb.last) { //after
+			output.n += output.runAppendInterval(vb)
+			j++
+		} else if va > vb.start {
+			if va < vb.last {
+				output.n += output.runAppendInterval(interval32{start: vb.start, last: va - 1})
+				vb.start = va + 1
+				i++
+				if vb.start > vb.last {
+					j++
+				}
+			} else if va > vb.last {
+				output.n += output.runAppendInterval(vb)
+				j++
+			} else { // va == vb.last
+				vb.last--
+				if vb.start < vb.last {
+					output.n += output.runAppendInterval(vb)
+				}
+				j++
+				i++
+			}
+
+		} else {
+			vb.start++
+			i++
+		}
+	}
+	if output.n < ArrayMaxSize {
+		output.runToArray()
+	} else if len(output.runs) > RunMaxSize {
+		output.runToBitmap()
+	}
+	return output
+}
+
+func xorCompare(x *stm) (r1 interval32, has_data bool) {
+	has_data = false
+	if !x.va_valid || !x.vb_valid {
+		if x.vb_valid {
+			x.vb_valid = false
+			r1 = x.vb
+			has_data = true
+			return
+		}
+		if x.va_valid {
+			x.va_valid = false
+			r1 = x.va
+			has_data = true
+			return
+		}
+		return
+	}
+
+	if x.va.last < x.vb.start { //a  before
+		x.va_valid = false
+		r1 = x.va
+		has_data = true
+	} else if x.vb.last < x.va.start { // b before
+		x.vb_valid = false
+		r1 = x.va
+		has_data = true
+	} else if x.va.start == x.vb.start && x.va.last == x.vb.last { // Equal
+		x.va_valid = false
+		x.vb_valid = false
+	} else if x.va.start <= x.vb.start && x.va.last >= x.vb.last { // b inside
+		x.vb_valid = false
+		if x.va.start != x.vb.start {
+			r1 = interval32{start: x.va.start, last: x.vb.start - 1}
+			has_data = true
+		}
+		x.va.start = x.vb.last + 1
+		if x.va.start > x.va.last {
+			x.va_valid = false
+		}
+
+	} else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //a inside
+		x.va_valid = false
+		if x.vb.start != x.va.start {
+			r1 = interval32{start: x.vb.start, last: x.va.start - 1}
+			has_data = true
+		}
+
+		x.vb.start = x.va.last + 1
+		if x.vb.start > x.vb.last {
+			x.vb_valid = false
+		}
+
+	} else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //a first overlap
+		x.va_valid = false
+		r1 = interval32{start: x.va.start, last: x.vb.start - 1}
+		has_data = true
+		x.vb.start = x.va.last + 1
+		if x.vb.start > x.vb.last {
+			x.vb_valid = false
+		}
+	} else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //b first overlap
+		x.vb_valid = false
+		r1 = interval32{start: x.vb.start, last: x.va.start - 1}
+		has_data = true
+		x.va.start = x.vb.last + 1
+		if x.va.start > x.va.last {
+			x.va_valid = false
+		}
+	}
+	return
+}
+
+type stm struct {
+	va_valid, vb_valid bool
+	va, vb             interval32
+}
+
+func xorRunRun(a, b *container) *container {
+	na, nb := len(a.runs), len(b.runs)
+	if na == 0 {
+		return b.clone()
+	}
+	if nb == 0 {
+		return a.clone()
+	}
+	output := &container{}
+
+	last_i, last_j := -1, -1
+
+	state := &stm{}
+
+	for i, j := 0, 0; i < na || j < nb; { //how to terminate?
+		if i < na && last_i != i {
+			state.va = a.runs[i]
+			state.va_valid = true
+		}
+
+		if j < nb && last_j != j {
+			state.vb = b.runs[j]
+			state.vb_valid = true
+		}
+		last_i, last_j = i, j
+
+		r1, ok := xorCompare(state)
+		if ok {
+			output.n += output.runAppendInterval(r1)
+		}
+		if !state.va_valid {
+			i++
+		}
+		if !state.vb_valid {
+			j++
+		}
+
+	}
+
+	if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
+		output.runToArray()
+	} else if len(output.runs) > RunMaxSize {
+		output.runToBitmap()
+	}
+	return output
+}
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index be2b07590..c9607f5a8 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -884,6 +884,36 @@ func TestArrayToRun(t *testing.T) {
 	}
 }
 
+func TestRunToArray(t *testing.T) {
+	a := &container{}
+	tests := []struct {
+		runs []interval32
+		exp  []uint32
+	}{
+		{
+			runs: []interval32{{start: 0, last: 0}},
+			exp:  []uint32{0},
+		},
+		{
+			runs: []interval32{{start: 0, last: 4}},
+			exp:  []uint32{0, 1, 2, 3, 4},
+		},
+		{
+			runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
+			exp:  []uint32{2, 5, 6, 7, 13, 14, 17},
+		},
+	}
+
+	for i, test := range tests {
+		a.runs = test.runs
+		a.n = len(test.exp)
+		a.runToArray()
+		if !reflect.DeepEqual(a.array, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array)
+		}
+	}
+}
+
 func TestBitmapZeroRange(t *testing.T) {
 	c := &container{bitmap: make([]uint64, bitmapN)}
 	tests := []struct {
@@ -1234,8 +1264,8 @@ func TestWriteReadArray(t *testing.T) {
 
 func TestWriteReadBitmap(t *testing.T) {
 	// create bitmap containing > 4096 bits
-	cb := &container{bitmap: make([]uint64, bitmapN), n: 129*32}
-	for i := 0; i < 129 ; i++ {
+	cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32}
+	for i := 0; i < 129; i++ {
 		cb.bitmap[i] = 0x5555555555555555
 	}
 	bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}}
@@ -1249,7 +1279,7 @@ func TestWriteReadBitmap(t *testing.T) {
 }
 
 func TestWriteReadRun(t *testing.T) {
-	cr := &container{runs: []interval32{{start: 3, last: 13}, {start: 100, last:109}}, n: 20}
+	cr := &container{runs: []interval32{{start: 3, last: 13}, {start: 100, last: 109}}, n: 20}
 	br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}}
 	br2 := &Bitmap{}
 	var buf bytes.Buffer
@@ -1259,3 +1289,73 @@ func TestWriteReadRun(t *testing.T) {
 		t.Fatalf("run test expected %x, but got %x", cr.runs, br2.containers[0].runs)
 	}
 }
+
+func TestXorArrayRun(t *testing.T) {
+	a := &container{array: []uint32{1, 5, 10, 11, 12}}
+	b := &container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	exp := []uint32{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}
+
+	ret := xorArrayRun(a, b)
+	if !reflect.DeepEqual(ret.array, exp) {
+		t.Fatalf("test expected %v, but got %v", exp, ret.array)
+	}
+}
+
+func TestXorRunRun(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		aruns []interval32
+		bruns []interval32
+		exp   []interval32
+	}{
+		{
+			aruns: []interval32{},
+			bruns: []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 5, last: 10}},
+		},
+		{
+			aruns: []interval32{{start: 5, last: 12}},
+			bruns: []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 11, last: 12}},
+		},
+		{
+			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			bruns: []interval32{{start: 5, last: 10}},
+			exp:   []interval32{{start: 1, last: 3}, {start: 6, last: 6}, {start: 11, last: 12}},
+		},
+		{
+			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			bruns: []interval32{{start: 2, last: 65535}},
+			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
+		},
+		{
+			aruns: []interval32{{start: 2, last: 65535}},
+			bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
+		},
+		{
+			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			bruns: []interval32{{start: 0, last: 65535}},
+			exp:   []interval32{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
+		},
+		{
+			aruns: []interval32{{start: 0, last: 65535}},
+			bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			exp:   []interval32{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
+		},
+		{
+			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}},
+			bruns: []interval32{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}},
+			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}},
+		},
+	}
+	for i, test := range tests {
+		a.runs = test.aruns
+		b.runs = test.bruns
+		ret := xorRunRun(a, b)
+		if !reflect.DeepEqual(ret.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
+		}
+	}
+}

From f4f07c0aa2ac6cc7fd7ed4b3c3188a18c9f84204 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Sat, 10 Jun 2017 08:17:53 -0500
Subject: [PATCH 039/100] extended xor tests; updated comments

---
 roaring/roaring.go               | 24 ++++++++++++++----------
 roaring/roaring_internal_test.go | 30 ++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 2da7511b1..25c8c6d53 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2911,6 +2911,7 @@ func assert(condition bool, format string, a ...interface{}) {
 	}
 }
 
+// xorArrayRun computes the exclusive or of an array and a run container.
 func xorArrayRun(a, b *container) *container {
 	output := &container{}
 	na, nb := len(a.array), len(b.runs)
@@ -2966,7 +2967,8 @@ func xorArrayRun(a, b *container) *container {
 	return output
 }
 
-func xorCompare(x *stm) (r1 interval32, has_data bool) {
+// xorCompare computes first exclusive run between two runs.
+func xorCompare(x *xorstm) (r1 interval32, has_data bool) {
 	has_data = false
 	if !x.va_valid || !x.vb_valid {
 		if x.vb_valid {
@@ -2984,18 +2986,18 @@ func xorCompare(x *stm) (r1 interval32, has_data bool) {
 		return
 	}
 
-	if x.va.last < x.vb.start { //a  before
+	if x.va.last < x.vb.start { //va  before
 		x.va_valid = false
 		r1 = x.va
 		has_data = true
-	} else if x.vb.last < x.va.start { // b before
+	} else if x.vb.last < x.va.start { //vb before
 		x.vb_valid = false
 		r1 = x.va
 		has_data = true
 	} else if x.va.start == x.vb.start && x.va.last == x.vb.last { // Equal
 		x.va_valid = false
 		x.vb_valid = false
-	} else if x.va.start <= x.vb.start && x.va.last >= x.vb.last { // b inside
+	} else if x.va.start <= x.vb.start && x.va.last >= x.vb.last { //vb inside
 		x.vb_valid = false
 		if x.va.start != x.vb.start {
 			r1 = interval32{start: x.va.start, last: x.vb.start - 1}
@@ -3006,7 +3008,7 @@ func xorCompare(x *stm) (r1 interval32, has_data bool) {
 			x.va_valid = false
 		}
 
-	} else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //a inside
+	} else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //va inside
 		x.va_valid = false
 		if x.vb.start != x.va.start {
 			r1 = interval32{start: x.vb.start, last: x.va.start - 1}
@@ -3018,7 +3020,7 @@ func xorCompare(x *stm) (r1 interval32, has_data bool) {
 			x.vb_valid = false
 		}
 
-	} else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //a first overlap
+	} else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //va first overlap
 		x.va_valid = false
 		r1 = interval32{start: x.va.start, last: x.vb.start - 1}
 		has_data = true
@@ -3026,7 +3028,7 @@ func xorCompare(x *stm) (r1 interval32, has_data bool) {
 		if x.vb.start > x.vb.last {
 			x.vb_valid = false
 		}
-	} else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //b first overlap
+	} else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //vb first overlap
 		x.vb_valid = false
 		r1 = interval32{start: x.vb.start, last: x.va.start - 1}
 		has_data = true
@@ -3038,11 +3040,13 @@ func xorCompare(x *stm) (r1 interval32, has_data bool) {
 	return
 }
 
-type stm struct {
+//stm  is state machine used to "xor" iterate over runs.
+type xorstm struct {
 	va_valid, vb_valid bool
 	va, vb             interval32
 }
 
+// xorRunRun computes the exclusive or of two run containers.
 func xorRunRun(a, b *container) *container {
 	na, nb := len(a.runs), len(b.runs)
 	if na == 0 {
@@ -3055,9 +3059,9 @@ func xorRunRun(a, b *container) *container {
 
 	last_i, last_j := -1, -1
 
-	state := &stm{}
+	state := &xorstm{}
 
-	for i, j := 0, 0; i < na || j < nb; { //how to terminate?
+	for i, j := 0, 0; i < na || j < nb; {
 		if i < na && last_i != i {
 			state.va = a.runs[i]
 			state.va_valid = true
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index c9607f5a8..deb92f34e 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1314,6 +1314,36 @@ func TestXorRunRun(t *testing.T) {
 			bruns: []interval32{{start: 5, last: 10}},
 			exp:   []interval32{{start: 5, last: 10}},
 		},
+		{
+			aruns: []interval32{{start: 0, last: 4}},
+			bruns: []interval32{{start: 6, last: 10}},
+			exp:   []interval32{{start: 0, last: 4}, {start: 6, last: 10}},
+		},
+		{
+			aruns: []interval32{{start: 0, last: 6}},
+			bruns: []interval32{{start: 4, last: 10}},
+			exp:   []interval32{{start: 0, last: 3}, {start: 7, last: 10}},
+		},
+		{
+			aruns: []interval32{{start: 4, last: 10}},
+			bruns: []interval32{{start: 0, last: 6}},
+			exp:   []interval32{{start: 0, last: 3}, {start: 7, last: 10}},
+		},
+		{
+			aruns: []interval32{{start: 0, last: 10}},
+			bruns: []interval32{{start: 0, last: 6}},
+			exp:   []interval32{{start: 7, last: 10}},
+		},
+		{
+			aruns: []interval32{{start: 0, last: 6}},
+			bruns: []interval32{{start: 0, last: 10}},
+			exp:   []interval32{{start: 7, last: 10}},
+		},
+		{
+			aruns: []interval32{{start: 0, last: 6}},
+			bruns: []interval32{{start: 0, last: 10}},
+			exp:   []interval32{{start: 7, last: 10}},
+		},
 		{
 			aruns: []interval32{{start: 5, last: 12}},
 			bruns: []interval32{{start: 5, last: 10}},

From bf7a1bc08a84640e452c795ae22069784244259e Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Mon, 12 Jun 2017 11:51:24 -0500
Subject: [PATCH 040/100] WIP bug fix, cleanup, additional tests

---
 fragment_test.go                 |  72 +++++++++++++++++++++
 roaring/roaring.go               | 105 ++++++++++++++++++++++++-------
 roaring/roaring_internal_test.go |  36 ++---------
 3 files changed, 159 insertions(+), 54 deletions(-)

diff --git a/fragment_test.go b/fragment_test.go
index f6b36f11e..9ed53f24b 100644
--- a/fragment_test.go
+++ b/fragment_test.go
@@ -17,6 +17,7 @@ package pilosa_test
 import (
 	"bytes"
 	"flag"
+	"fmt"
 	"io/ioutil"
 	"math"
 	"os"
@@ -248,6 +249,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
 	// Set bits on rows 0 - 999. Higher rows have higher bit counts.
 	for i := uint64(0); i < 1000; i++ {
 		for j := uint64(0); j < i; j++ {
+			//fmt.Printf("testfragment: setting %d, %d\n", i, j)
 			f.MustSetBits(i, j)
 		}
 	}
@@ -272,6 +274,45 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
 	}
 }
 
+// temporary test for catching op log bug
+func TestFragment_TopN_Intersect_Moderate(t *testing.T) {
+	f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
+	defer f.Close()
+
+	// Create an intersecting input row.
+	src := pilosa.NewBitmap(
+		80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
+		90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
+	)
+
+	// Set bits on rows 0 - 99. Higher rows have higher bit counts.
+	for i := uint64(0); i < 100; i++ {
+		for j := uint64(0); j < i; j++ {
+			//fmt.Printf("testfragment: setting %d, %d\n", i, j)
+			f.MustSetBits(i, j)
+		}
+	}
+	f.RecalculateCache()
+
+	// Retrieve top rows.
+	if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil {
+		t.Fatal(err)
+	} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
+		{ID: 99, Count: 19},
+		{ID: 98, Count: 18},
+		{ID: 97, Count: 17},
+		{ID: 96, Count: 16},
+		{ID: 95, Count: 15},
+		{ID: 94, Count: 14},
+		{ID: 93, Count: 13},
+		{ID: 92, Count: 12},
+		{ID: 91, Count: 11},
+		{ID: 90, Count: 10},
+	}) {
+		t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs))
+	}
+}
+
 // Ensure a fragment can return top rows when specified by ID.
 func TestFragment_TopN_IDs(t *testing.T) {
 	f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
@@ -789,3 +830,34 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
 		t.Fatalf("unexpected pair(1): %v", pairs[2])
 	}
 }
+
+func TestFragment_Snapshot_Run(t *testing.T) {
+	fmt.Printf("mustopenfragment\n")
+    f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+    defer f.Close()
+
+	fmt.Printf("\n---\nset bits\n")
+    // Set bits on the fragment.
+    for i := uint64(1); i < 3; i++ {
+        if _, err := f.SetBit(1000, i); err != nil {
+            t.Fatal(err)
+        }
+    }
+
+	fmt.Printf("\n---\nsnapshot\n")
+    // Snapshot bitmap and verify data.
+    if err := f.Snapshot(); err != nil {
+        t.Fatal(err)
+    } else if n := f.Row(1000).Count(); n != 2 {
+        t.Fatalf("unexpected count: %d", n)
+    }
+
+	fmt.Printf("\n---\nreopen\n")
+    // Close and reopen the fragment & verify the data.
+    if err := f.Reopen(); err != nil {
+        t.Fatal(err)
+    } else if n := f.Row(1000).Count(); n != 2 {
+        t.Fatalf("unexpected count (reopen): %d", n)
+    }
+}
+
diff --git a/roaring/roaring.go b/roaring/roaring.go
index 25c8c6d53..5cf82a348 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -26,9 +26,17 @@ import (
 )
 
 const (
-	// cookie is the first four bytes in a roaring bitmap file.
-	cookieNoRuns = uint32(12346)
-	cookie       = uint32(12347)
+	// magicNumber is an identifier, in bytes 0-1 of the file.
+	magicNumberNoRuns = uint32(12346)
+	magicNumber       = uint32(12347)
+
+	// storageVersion indicates the storage version, in bytes 2-3.
+	storageVersion = uint32(0)
+
+	// cookie is the first four bytes in a roaring bitmap file,
+	// formed by joining magicNumber and storageVersion
+	cookieNoRuns = magicNumberNoRuns<<16 + storageVersion
+	cookie       = magicNumber<<16 + storageVersion
 
 	// headerBaseSize is the size of the cookie and key count at the beginning of a file.
 	// Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8.
@@ -661,7 +669,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		// Map byte slice directly to the container data.
 		c := b.containers[i]
 		if c.n <= ArrayMaxSize {
-			if containsRuns && (runFlagBitset[uint8(i)/8]&(1<<uint8(i)%8)) != 0 {
+			if containsRuns && (runFlagBitset[uint8(i)/8]&(1<<uint8(i%8))) != 0 {
 				// Read runs.
 				runCount := binary.LittleEndian.Uint16(data[offset : offset+2])
 				c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+2]))[:runCount] // TODO verify
@@ -680,7 +688,6 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 			c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
 			opsOffset = int(offset) + len(c.bitmap)*8
 		}
-
 		// Verify container count on load.
 		// TODO: instead of commenting this out, we need to make it a configuration option
 		//count := c.count()
@@ -809,8 +816,8 @@ type BitmapInfo struct {
 
 // Iterator represents an iterator over a Bitmap.
 type Iterator struct {
-	bitmap *Bitmap
-	i, j   int
+	bitmap  *Bitmap
+	i, j, k int // i: container; j: array index, bit index, or run index; k:
 }
 
 // eof returns true if the iterator is at the end of the bitmap.
@@ -829,7 +836,8 @@ func (itr *Iterator) Seek(seek uint64) {
 
 	// Move to the correct value index inside the array container.
 	lb := lowbits(seek)
-	if c := itr.bitmap.containers[itr.i]; c.isArray() {
+	c := itr.bitmap.containers[itr.i]
+	if c.isArray() {
 		// Find index in the container.
 		itr.j = search32(c.array, lb)
 		if itr.j < 0 {
@@ -845,6 +853,15 @@ func (itr *Iterator) Seek(seek uint64) {
 		return
 	}
 
+	if c.isRun() {
+		// TODO work for seek!=0
+		itr.i, itr.j, itr.k = 0, 0, -1
+		if seek != 0 {
+			panic("cant seeek nonzero")
+		}
+		return
+	}
+
 	// If it's a bitmap container then move to index before the value and call next().
 	itr.j = int(lb) - 1
 }
@@ -868,6 +885,35 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
 			itr.j++
 			return itr.peek(), false
 		}
+
+		if c.isRun() {
+			if itr.j >= len(c.runs)-1 {
+				r := c.runs[itr.j]
+				runLength := int(r.last - r.start)
+
+				if itr.k >= runLength {
+					itr.i++
+					itr.j = -1
+					continue
+				} else {
+					itr.k++
+					return itr.peek(), false
+				}
+			}
+
+			r := c.runs[itr.j]
+			runLength := int(r.last - r.start)
+
+			if itr.k >= runLength {
+				itr.k = 0
+				itr.j++
+			} else {
+				itr.k++
+			}
+
+			return itr.peek(), false
+		}
+
 		// Move to the next possible index in the bitmap container.
 		itr.j++
 
@@ -904,6 +950,9 @@ func (itr *Iterator) peek() uint64 {
 	if c.isArray() {
 		return uint64(key)<<16 | uint64(c.array[itr.j])
 	}
+	if c.isRun() {
+		return uint64(key)<<16 | uint64(c.runs[itr.j].start+uint32(itr.k))
+	}
 	return uint64(key)<<16 | uint64(itr.j)
 }
 
@@ -1002,10 +1051,12 @@ func (c *container) isArray() bool {
 	return c.bitmap == nil && c.runs == nil
 }
 
+// isBitmap returns true if the container is a bitmap container
 func (c *container) isBitmap() bool {
 	return c.array == nil && c.runs == nil
 }
 
+// isRun returns true if the container is a run-length-encoded container
 func (c *container) isRun() bool {
 	return c.array == nil && c.bitmap == nil
 }
@@ -1582,7 +1633,7 @@ func (c *container) size() int {
 	if c.isArray() {
 		return len(c.array) * 4
 	} else if c.isRun() {
-		return len(c.runs) * 8
+		return len(c.runs)*8 + 2
 	} else {
 		return len(c.bitmap) * 8
 	}
@@ -1597,7 +1648,7 @@ func (c *container) info() ContainerInfo {
 		info.Alloc = len(c.array) * 4
 	} else if c.isRun() {
 		info.Type = "run"
-		info.Alloc = len(c.runs) * 8
+		info.Alloc = len(c.runs)*8 + 2
 	} else {
 		info.Type = "bitmap"
 		info.Alloc = len(c.bitmap) * 8
@@ -2323,26 +2374,28 @@ func differenceArrayArray(a, b *container) *container {
 func differenceArrayRun(a, b *container) *container {
 	// func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
 
-	if b.n == 0 {
+	if a.n == 0 || b.n == 0 {
 		return a.clone()
 	}
 
 	output := &container{array: make([]uint32, 0, a.n)}
 	// cardinality upper bound: card(A)
 
-	i := 0  // array index
-	j := 0  // run index
+	i := 0 // array index
+	j := 0 // run index
 
 	// keep all array elements before beginning of runs
-	for ; i < b.runs[j].start; i++ {
+	for ; i < int(b.runs[j].start); i++ {
 		output.array = append(output.array, a.array[i])
 	}
 
 	// handle overlap
 	for ; i < a.n; i++ {
+		// if array element in run, keep
 		if !(a.array[i] >= b.runs[j].start && a.array[i] <= b.runs[j].last) {
 			output.array = append(output.array, a.array[i])
 		}
+		// update current run
 		if i >= int(b.runs[j].last) {
 			j++
 			if j == len(b.runs) {
@@ -2354,7 +2407,7 @@ func differenceArrayRun(a, b *container) *container {
 
 	// keep all array elements after end of runs
 	output.array = append(output.array, a.array[i:]...)
-	
+
 	return output
 }
 
@@ -2372,27 +2425,33 @@ func differenceBitmapRun(a, b *container) *container {
 
 func differenceRunArray(a, b *container) *container {
 	// TODO
+	if a.n == 0 || b.n == 0 {
+		return a.clone()
+	}
+
 	output := &container{runs: make([]interval32, 0, a.n)}
 	return output
 }
 
 func differenceRunBitmap(a, b *container) *container {
+	// TODO
+	if a.n == 0 || b.n == 0 {
+		return a.clone()
+	}
+
 	output := &container{runs: make([]interval32, 0, a.n)}
 	itr := newBufIterator(newBitmapIterator(b.bitmap))
 
-	fmt.Printf("\ndifferenceRunBitmap\n")
 	for i := 0; ; {
 		vb, eof := itr.next()
-		if eof || {
+		if eof {
 			break
 		}
-
-		fmt.Println(i, vb, eof)
+		fmt.Println(i, vb)
 
 		i++
 	}
 
-
 	return output
 }
 
@@ -2403,8 +2462,8 @@ func differenceRunRun(a, b *container) *container {
 		return a.clone()
 	}
 
-	apos := 0  // current a-run index
-	bpos := 0  // current b-run index
+	apos := 0 // current a-run index
+	bpos := 0 // current b-run index
 	astart := a.runs[apos].start
 	alast := a.runs[apos].last
 	bstart := b.runs[bpos].start
@@ -2412,7 +2471,7 @@ func differenceRunRun(a, b *container) *container {
 	alen := len(a.runs)
 	blen := len(b.runs)
 
-	output := &container{runs: make([]interval32, 0, alen+blen)}  // TODO allocate max then truncate? or something else
+	output := &container{runs: make([]interval32, 0, alen+blen)} // TODO allocate max then truncate? or something else
 	// cardinality upper bound: sum of number of runs
 	// each B-run could split an A-run in two, up to len(b.runs) times
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index deb92f34e..526d4f194 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -996,36 +996,6 @@ func TestUnionBitmapRun(t *testing.T) {
 	}
 }
 
-func TestRunToArray(t *testing.T) {
-	a := &container{}
-	tests := []struct {
-		runs []interval32
-		exp  []uint32
-	}{
-		{
-			runs: []interval32{{start: 0, last: 0}},
-			exp:  []uint32{0},
-		},
-		{
-			runs: []interval32{{start: 0, last: 4}},
-			exp:  []uint32{0, 1, 2, 3, 4},
-		},
-		{
-			runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
-			exp:  []uint32{2, 5, 6, 7, 13, 14, 17},
-		},
-	}
-
-	for i, test := range tests {
-		a.runs = test.runs
-		a.n = len(test.exp)
-		a.runToArray()
-		if !reflect.DeepEqual(a.array, test.exp) {
-			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array)
-		}
-	}
-}
-
 func TestBitmapCountRuns(t *testing.T) {
 	c := &container{bitmap: make([]uint64, bitmapN)}
 	tests := []struct {
@@ -1133,6 +1103,7 @@ func TestDifferenceArrayRun(t *testing.T) {
 	}
 	for i, test := range tests {
 		a.array = test.array
+		a.n = len(a.array)
 		b.runs = test.runs
 		b.n = b.runCountRange(0, 100)
 		ret := differenceArrayRun(a, b)
@@ -1160,6 +1131,7 @@ func TestDifferenceRunArray(t *testing.T) {
 		a.runs = test.runs
 		a.n = a.runCountRange(0, 100)
 		b.array = test.array
+		b.n = len(b.array)
 		ret := differenceRunArray(a, b)
 		if !reflect.DeepEqual(ret.array, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
@@ -1187,6 +1159,7 @@ func TestDifferenceRunBitmap(t *testing.T) {
 		for i, v := range test.bitmap {
 			b.bitmap[i] = v
 		}
+		b.n = b.bitmapCountRange(0, 100)
 		ret := differenceRunBitmap(a, b)
 		if !reflect.DeepEqual(ret.runs, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
@@ -1212,11 +1185,12 @@ func TestDifferenceBitmapRun(t *testing.T) {
 		for i, v := range test.bitmap {
 			a.bitmap[i] = v
 		}
+		a.n = a.bitmapCountRange(0, 100)
 		b.runs = test.runs
 		b.n = b.runCountRange(0, 100)
 		ret := differenceBitmapRun(a, b)
 		if !reflect.DeepEqual(ret.bitmap[:len(test.exp)], test.exp) {
-			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap[:len(test.exp)])
+			t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.bitmap[:len(test.exp)])
 		}
 	}
 }

From 93a7320cd97834264864d6fc9a164d02b401cc59 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Mon, 12 Jun 2017 15:59:26 -0500
Subject: [PATCH 041/100] bitmap/run support for rle

---
 roaring/roaring.go               | 41 ++++++++++++++
 roaring/roaring_internal_test.go | 93 ++++++++++++++++++++++++++++++++
 2 files changed, 134 insertions(+)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 5cf82a348..702885834 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2252,6 +2252,32 @@ func (c *container) bitmapSetRange(i, j uint64) {
 	}
 }
 
+// xor's all bits in [i, j] with all true (inclusive) (c must be a bitmap container).
+func (c *container) bitmapXorRange(i, j uint64) {
+	j += 1
+	x := i / 64
+	y := (j - 1) / 64
+	var X uint64 = maxBitmap << (i % 64)
+	var Y uint64 = maxBitmap >> (64 - (j % 64))
+	if x == y {
+		cnt := popcnt(c.bitmap[x])
+		c.bitmap[x] ^= (X & Y) //// flip
+		c.n += int(popcnt(c.bitmap[x]) - cnt)
+	} else {
+		cnt := popcnt(c.bitmap[x])
+		c.bitmap[x] ^= X
+		c.n += int(popcnt(c.bitmap[x]) - cnt)
+		for i := x + 1; i < y; i++ {
+			cnt = popcnt(c.bitmap[i])
+			c.bitmap[i] ^= maxBitmap
+			c.n += int(popcnt(c.bitmap[i]) - cnt)
+		}
+		cnt = popcnt(c.bitmap[y])
+		c.bitmap[y] ^= Y
+		c.n += int(popcnt(c.bitmap[y]) - cnt)
+	}
+}
+
 // zeroes all bits in [i, j] (inclusive) (c must be a bitmap container)
 func (c *container) bitmapZeroRange(i, j uint64) {
 	j += 1
@@ -3152,3 +3178,18 @@ func xorRunRun(a, b *container) *container {
 	}
 	return output
 }
+
+// xorRunRun computes the exclusive or of a bitmap and a run container.
+func xorBitmapRun(a, b *container) *container {
+	output := a.clone()
+	for j := 0; j < len(b.runs); j++ {
+		output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last))
+	}
+
+	if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
+		output.runToArray()
+	} else if len(output.runs) > RunMaxSize {
+		output.runToBitmap()
+	}
+	return output
+}
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 526d4f194..1745e0c97 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1363,3 +1363,96 @@ func TestXorRunRun(t *testing.T) {
 		}
 	}
 }
+
+func TestBitmapFlipRange(t *testing.T) {
+	c := &container{bitmap: make([]uint64, bitmapN)}
+	tests := []struct {
+		bitmap []uint64
+		start  uint64
+		last   uint64
+		exp    []uint64
+		expN   int
+	}{
+		{
+			bitmap: []uint64{0x0000000000000000},
+			start:  0,
+			last:   2,
+			exp:    []uint64{0x0000000000000007},
+			expN:   3,
+		},
+		{
+			bitmap: []uint64{0xF1},
+			start:  4,
+			last:   8,
+			exp:    []uint64{0x101},
+			expN:   2,
+		},
+		{
+			bitmap: []uint64{0xAA},
+			start:  0,
+			last:   7,
+			exp:    []uint64{0x55},
+			expN:   4,
+		},
+		{
+			bitmap: []uint64{0x0, 0x0000000000000000, 0x0000000000000000},
+			start:  63,
+			last:   128,
+			exp:    []uint64{0x8000000000000000, 0xFFFFFFFFFFFFFFFF, 0x000000000000001},
+			expN:   66,
+		},
+		{
+			bitmap: []uint64{0x0, 0x00000000000000FF, 0x0000000000000000},
+			start:  63,
+			last:   128,
+			exp:    []uint64{0x8000000000000000, 0xFFFFFFFFFFFFFF00, 0x000000000000001},
+			expN:   58,
+		},
+		{
+			bitmap: []uint64{0x0, 0x0, 0x0},
+			start:  129,
+			last:   131,
+			exp:    []uint64{0x0000000000000000, 0x0000000000000000, 0x00000000000000E},
+			expN:   3,
+		},
+	}
+
+	for i, test := range tests {
+		for i, v := range test.bitmap {
+			c.bitmap[i] = v
+		}
+		c.n = c.countRange(0, 65535)
+		c.bitmapXorRange(test.start, test.last)
+		if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) {
+			t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)])
+		}
+		if test.expN != c.n {
+			t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, c.n)
+		}
+	}
+}
+
+func TestXorBitmapRun(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	tests := []struct {
+		bitmap []uint64
+		runs   []interval32
+		exp    []uint64
+	}{
+		{bitmap: []uint64{0x0, 0x0, 0x0},
+			runs: []interval32{{start: 129, last: 131}},
+			exp:  []uint64{0x0, 0x0, 0x00000000000000E},
+		},
+	}
+	for i, test := range tests {
+		a.bitmap = test.bitmap
+		b.runs = test.runs
+		ret := xorBitmapRun(a, b)
+
+		if !reflect.DeepEqual(ret.bitmap, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap)
+		}
+	}
+
+}

From bbf6d78169d4cc014926c4d87c9838db824e1923 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Mon, 12 Jun 2017 16:01:44 -0500
Subject: [PATCH 042/100] corrected testname for bitmapXorRange

---
 roaring/roaring_internal_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 1745e0c97..8c35dc68a 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1364,7 +1364,7 @@ func TestXorRunRun(t *testing.T) {
 	}
 }
 
-func TestBitmapFlipRange(t *testing.T) {
+func TestBitmapXorRange(t *testing.T) {
 	c := &container{bitmap: make([]uint64, bitmapN)}
 	tests := []struct {
 		bitmap []uint64

From e72ce82a498b0c51c9a8f368a276d917aaaa1b21 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Tue, 13 Jun 2017 13:36:39 -0500
Subject: [PATCH 043/100] Fix bug - all tests passing

---
 fragment_test.go                 |  7 -------
 roaring/roaring.go               | 16 ++++++++++------
 roaring/roaring_internal_test.go |  2 ++
 3 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/fragment_test.go b/fragment_test.go
index 9ed53f24b..13cc4e808 100644
--- a/fragment_test.go
+++ b/fragment_test.go
@@ -17,7 +17,6 @@ package pilosa_test
 import (
 	"bytes"
 	"flag"
-	"fmt"
 	"io/ioutil"
 	"math"
 	"os"
@@ -249,7 +248,6 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
 	// Set bits on rows 0 - 999. Higher rows have higher bit counts.
 	for i := uint64(0); i < 1000; i++ {
 		for j := uint64(0); j < i; j++ {
-			//fmt.Printf("testfragment: setting %d, %d\n", i, j)
 			f.MustSetBits(i, j)
 		}
 	}
@@ -288,7 +286,6 @@ func TestFragment_TopN_Intersect_Moderate(t *testing.T) {
 	// Set bits on rows 0 - 99. Higher rows have higher bit counts.
 	for i := uint64(0); i < 100; i++ {
 		for j := uint64(0); j < i; j++ {
-			//fmt.Printf("testfragment: setting %d, %d\n", i, j)
 			f.MustSetBits(i, j)
 		}
 	}
@@ -832,11 +829,9 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
 }
 
 func TestFragment_Snapshot_Run(t *testing.T) {
-	fmt.Printf("mustopenfragment\n")
     f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
     defer f.Close()
 
-	fmt.Printf("\n---\nset bits\n")
     // Set bits on the fragment.
     for i := uint64(1); i < 3; i++ {
         if _, err := f.SetBit(1000, i); err != nil {
@@ -844,7 +839,6 @@ func TestFragment_Snapshot_Run(t *testing.T) {
         }
     }
 
-	fmt.Printf("\n---\nsnapshot\n")
     // Snapshot bitmap and verify data.
     if err := f.Snapshot(); err != nil {
         t.Fatal(err)
@@ -852,7 +846,6 @@ func TestFragment_Snapshot_Run(t *testing.T) {
         t.Fatalf("unexpected count: %d", n)
     }
 
-	fmt.Printf("\n---\nreopen\n")
     // Close and reopen the fragment & verify the data.
     if err := f.Reopen(); err != nil {
         t.Fatal(err)
diff --git a/roaring/roaring.go b/roaring/roaring.go
index 702885834..80a089d87 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -530,14 +530,14 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 	// Create bitset indicating runs, record whether any runs present.
 	containsRuns := false
 	runFlagBitset := make([]uint8, (containerCount+7)/8) // TODO verify size
-	var k uint8 = 0
+	k := 0
 	for _, c := range b.containers {
 		if c.n == 0 {
 			continue
 		}
 		if c.isRun() {
 			containsRuns = true
-			runFlagBitset[k/8] |= (1 << (k % 8)) // TODO verify
+			runFlagBitset[k/8] |= (1 << uint(k%8)) // TODO verify
 		}
 		k++
 	}
@@ -669,7 +669,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		// Map byte slice directly to the container data.
 		c := b.containers[i]
 		if c.n <= ArrayMaxSize {
-			if containsRuns && (runFlagBitset[uint8(i)/8]&(1<<uint8(i%8))) != 0 {
+			if containsRuns && (runFlagBitset[i/8]&(1<<uint(i%8))) != 0 {
 				// Read runs.
 				runCount := binary.LittleEndian.Uint16(data[offset : offset+2])
 				c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+2]))[:runCount] // TODO verify
@@ -1373,7 +1373,7 @@ func (c *container) bitmapRemove(v uint32) bool {
 	c.unmap()
 
 	// Lower count and remove element.
-	c.n--
+	// c.n-- // TODO removed this - test it
 	c.bitmap[v/64] &^= (uint64(1) << (v % 64))
 
 	// Convert to array if we go below the threshold.
@@ -1489,7 +1489,7 @@ func (c *container) runToBitmap() {
 
 // bitmapToRun converts from bitmap format to RLE format.
 func (c *container) bitmapToRun() {
-	numRuns := c.n // TODO compute properly
+	numRuns := c.bitmapCountRuns() // TODO test
 	c.runs = make([]interval32, 0, numRuns)
 	// TODO return early if no runs
 
@@ -1537,7 +1537,7 @@ func (c *container) bitmapToRun() {
 
 // arrayToRun converts from array format to RLE format.
 func (c *container) arrayToRun() {
-	c.runs = make([]interval32, 0, c.n) // what capacity to use?
+	c.runs = make([]interval32, 0, c.n) // TODO what capacity to use?
 	start := c.array[0]
 	for i, v := range c.array[1:] {
 		if v-c.array[i] > 1 {
@@ -1618,6 +1618,9 @@ func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) {
 }
 
 func (c *container) runWriteTo(w io.Writer) (n int64, err error) {
+	if len(c.runs) == 0 {
+		return 0, nil
+	}
 	// TODO according to spec this should be [16-bit-runcount, start0, len0, start1, len1, ...]
 	// not sure what difference len vs last makes
 	err = binary.Write(w, binary.LittleEndian, uint16(len(c.runs)))
@@ -1672,6 +1675,7 @@ func (c *container) check() error {
 	var a ErrorList
 
 	if c.n <= ArrayMaxSize {
+		// TODO check run here
 		if len(c.array) != c.n {
 			a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(c.array), c.n))
 		}
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 8c35dc68a..c3d0c5ac8 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1113,6 +1113,7 @@ func TestDifferenceArrayRun(t *testing.T) {
 	}
 }
 
+/*
 func TestDifferenceRunArray(t *testing.T) {
 	a := &container{}
 	b := &container{}
@@ -1166,6 +1167,7 @@ func TestDifferenceRunBitmap(t *testing.T) {
 		}
 	}
 }
+*/
 
 func TestDifferenceBitmapRun(t *testing.T) {
 	a := &container{bitmap: make([]uint64, bitmapN)}

From 65b19168b39c07444c5958cb3546778cd9aeae49 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Tue, 13 Jun 2017 13:42:23 -0500
Subject: [PATCH 044/100] add test for bitcount test on remove

---
 roaring/roaring_test.go | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index d1bb609e2..07fd99d84 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -325,10 +325,18 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) {
 		m := make(map[uint64]struct{})
 
 		// Add values to the bitmap and set.
+		manual_count := uint64(0)
 		for _, v := range a {
-			bm.Add(v)
+			new_bit, _ := bm.Add(v)
+			if new_bit {
+				manual_count++
+			}
 			m[v] = struct{}{}
 		}
+		//check count
+		if manual_count != bm.Count() {
+			t.Fatalf("expected bitmap Add count to be: %d got: %d", manual_count, bm.Count())
+		}
 
 		// Verify existence.
 		for _, v := range a {
@@ -352,8 +360,16 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) {
 		}
 
 		// Remove all values in random order.
+		fmt.Println(len(a))
 		for _, i := range rand.Perm(len(a)) {
-			bm.Remove(a[i])
+			removed, _ := bm.Remove(a[i])
+			if removed {
+				manual_count--
+			}
+			//check count
+			if manual_count != bm.Count() {
+				t.Fatalf("expected bitmap Remove count to be: %d got: %d", manual_count, bm.Count())
+			}
 		}
 
 		// Verify all values have been removed.

From d3eef0d3eb7d70e4f368250c4b6b0b7af6c2ce36 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Tue, 13 Jun 2017 15:13:44 -0500
Subject: [PATCH 045/100] Cleanup comments and add replace some literals with
 consts

---
 roaring/roaring.go | 93 +++++++++++++++++++++++++++-------------------
 1 file changed, 54 insertions(+), 39 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 80a089d87..a752670c3 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -42,6 +42,13 @@ const (
 	// Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8.
 	headerBaseSize = 4 + 4
 
+	// runCountHeaderSize is the size in bytes of the run count stored
+	// at the beginning of every serialized run container.
+	runCountHeaderSize = 2
+
+	// interval32Size is the size of a single run in a container.runs.
+	interval32Size = 8
+
 	// bitmapN is the number of values in a container.bitmap.
 	bitmapN = (1 << 16) / 64
 
@@ -52,7 +59,7 @@ const (
 // Bitmap represents a roaring bitmap.
 type Bitmap struct {
 	keys       []uint64     // keys for containers
-	containers []*container // array and bitmap containers
+	containers []*container // array, bitmap and RLE containers
 
 	// Number of operations written to the writer.
 	opN int
@@ -529,7 +536,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 
 	// Create bitset indicating runs, record whether any runs present.
 	containsRuns := false
-	runFlagBitset := make([]uint8, (containerCount+7)/8) // TODO verify size
+	runFlagBitset := make([]uint8, (containerCount+7)/8)
 	k := 0
 	for _, c := range b.containers {
 		if c.n == 0 {
@@ -537,7 +544,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 		}
 		if c.isRun() {
 			containsRuns = true
-			runFlagBitset[k/8] |= (1 << uint(k%8)) // TODO verify
+			runFlagBitset[k/8] |= (1 << uint(k%8))
 		}
 		k++
 	}
@@ -550,9 +557,11 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 	}
 
 	// Build header before writing individual container blocks.
+	// Metadata for each container is 4+8+4 = sizeof(key) + sizeof(cardinality) + sizeof(file offset)
 	buf := make([]byte, headerSize+(containerCount*(4+8+4)))
+
+	// Cookie header section.
 	binary.LittleEndian.PutUint32(buf[0:], thisCookie)
-	// TODO supposed to be 2 bytes, but doesnt make sense for >65535 containers
 	binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount))
 
 	if containsRuns {
@@ -563,7 +572,8 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 	}
 
 	empty := 0
-	// Encode keys and cardinality.
+	// Descriptive header section: encode keys and cardinality.
+	// Key and cardinality are stored interleaved here, 12 bytes per container.
 	for i, key := range b.keys {
 		c := b.containers[i]
 
@@ -579,8 +589,8 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 		}
 	}
 
-	// TODO this should now be done conditionally
-	// Write the offset for each container block.
+	// Offset header section: write the offset for each container block.
+	// 4 bytes per container.
 	offset := uint32(len(buf))
 	empty = 0
 	for i, c := range b.containers {
@@ -600,7 +610,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 		return n, err
 	}
 
-	// Write each container block.
+	// Container storage section: write each container block.
 	for _, c := range b.containers {
 		if c.n > 0 {
 			nn, err := c.WriteTo(w)
@@ -620,7 +630,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		return errors.New("data too small")
 	}
 
-	// Verify the first 4 bytes are a valid cookie.
+	// Verify the first sizeof(cookie)=4 bytes are a valid cookie.
 	v := binary.LittleEndian.Uint32(data[0:4])
 	containsRuns := false
 	if v == cookieNoRuns {
@@ -631,14 +641,14 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		return errors.New("invalid roaring file")
 	}
 
-	// Read key count.
+	// Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)).
 	keyN := binary.LittleEndian.Uint32(data[4:8])
 	b.keys = make([]uint64, keyN)
 	b.containers = make([]*container, keyN)
 
 	headerSize := headerBaseSize
 
-	runFlagBitset := make([]uint8, (keyN+7)/8) // TODO verify size
+	runFlagBitset := make([]uint8, (keyN+7)/8)
 	if containsRuns {
 		// Read runFlag bitset.
 		for i := 0; i < len(runFlagBitset); i++ {
@@ -647,7 +657,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		headerSize += len(runFlagBitset)
 	}
 
-	// Read container key headers.
+	// Descriptive header section: Read container keys and cardinalities.
 	for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
 		b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
 		b.containers[i] = &container{
@@ -671,9 +681,9 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		if c.n <= ArrayMaxSize {
 			if containsRuns && (runFlagBitset[i/8]&(1<<uint(i%8))) != 0 {
 				// Read runs.
-				runCount := binary.LittleEndian.Uint16(data[offset : offset+2])
-				c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+2]))[:runCount] // TODO verify
-				opsOffset = int(offset) + 2 + len(c.runs)*8                                   // TODO verify
+				runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
+				c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount]
+				opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval32Size
 			} else {
 				// Read array.
 				c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
@@ -681,12 +691,12 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 				//for _, v := range c.array {
 				//    assert(lowbits(uint64(v)) == v, "array value out of range: %d", v)
 				//}
-				opsOffset = int(offset) + len(c.array)*4
+				opsOffset = int(offset) + len(c.array)*4 // sizeof(uint32)
 			}
 		} else {
 			// Read bitmap.
 			c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
-			opsOffset = int(offset) + len(c.bitmap)*8
+			opsOffset = int(offset) + len(c.bitmap)*8 // sizeof(uint64)
 		}
 		// Verify container count on load.
 		// TODO: instead of commenting this out, we need to make it a configuration option
@@ -1014,16 +1024,16 @@ const RunMaxSize = 2048
 
 // container represents a container for uint32 integers.
 //
-// These are used for storing the low bits. Containers are separated into two
+// These are used for storing the low bits. Containers are separated into three
 // types depending on cardinality. For containers with less than 4,096 values,
-// an array container is used. For containers with more than 4,096 values,
-// the values are encoded into bitmaps.
+// an array or RLE container is used, depending on the contents. For containers
+// with more than 4,096 values, the values are encoded into bitmaps.
 type container struct {
-	n      int      // number of integers in container
-	array  []uint32 // used for array containers
-	bitmap []uint64 // used for bitmap containers
-	runs   []interval32
-	mapped bool // mapped directly to a byte slice when true
+	n      int          // number of integers in container
+	array  []uint32     // used for array containers
+	bitmap []uint64     // used for bitmap containers
+	runs   []interval32 // used for RLE containers
+	mapped bool         // mapped directly to a byte slice when true
 }
 
 type interval32 struct {
@@ -1328,6 +1338,7 @@ func (c *container) bitmapContains(v uint32) bool {
 }
 
 func (c *container) runContains(v uint32) bool {
+	// TODO binary search
 	for _, iv := range c.runs {
 		if v > iv.last {
 			continue
@@ -1340,7 +1351,7 @@ func (c *container) runContains(v uint32) bool {
 	return false
 }
 
-// remove adds a value to the container.
+// remove removes a value from the container.
 func (c *container) remove(v uint32) (removed bool) {
 	if c.isArray() {
 		removed = c.arrayRemove(v)
@@ -1384,6 +1395,7 @@ func (c *container) bitmapRemove(v uint32) bool {
 }
 
 func (c *container) runRemove(v uint32) bool {
+	// TODO binary search
 	for i, iv := range c.runs {
 		if v <= iv.last {
 			if v < iv.start {
@@ -1537,7 +1549,8 @@ func (c *container) bitmapToRun() {
 
 // arrayToRun converts from array format to RLE format.
 func (c *container) arrayToRun() {
-	c.runs = make([]interval32, 0, c.n) // TODO what capacity to use?
+	numRuns := c.arrayCountRuns() // TODO test
+	c.runs = make([]interval32, 0, numRuns)
 	start := c.array[0]
 	for i, v := range c.array[1:] {
 		if v-c.array[i] > 1 {
@@ -1608,11 +1621,13 @@ func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) {
 	//  	assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v)
 	//}
 
+	// Write sizeof(uint32) * cardinality bytes.
 	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:4*c.n])
 	return int64(nn), err
 }
 
 func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) {
+	// Write sizeof(uint64) * bitmapN bytes.
 	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.bitmap[0]))[:(8 * bitmapN)])
 	return int64(nn), err
 }
@@ -1621,24 +1636,23 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) {
 	if len(c.runs) == 0 {
 		return 0, nil
 	}
-	// TODO according to spec this should be [16-bit-runcount, start0, len0, start1, len1, ...]
-	// not sure what difference len vs last makes
+	// Write sizeof(interval32) * runCount bytes.
 	err = binary.Write(w, binary.LittleEndian, uint16(len(c.runs)))
 	if err != nil {
 		return 0, err
 	}
-	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:8*len(c.runs)])
-	return int64(2 + nn), err
+	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:interval32Size*len(c.runs)])
+	return int64(runCountHeaderSize + nn), err
 }
 
 // size returns the encoded size of the container, in bytes.
 func (c *container) size() int {
 	if c.isArray() {
-		return len(c.array) * 4
+		return len(c.array) * 4 // sizeof(uint32)
 	} else if c.isRun() {
-		return len(c.runs)*8 + 2
+		return len(c.runs)*interval32Size + runCountHeaderSize
 	} else {
-		return len(c.bitmap) * 8
+		return len(c.bitmap) * 8 // sizeof(uint64)
 	}
 }
 
@@ -1648,13 +1662,13 @@ func (c *container) info() ContainerInfo {
 
 	if c.isArray() {
 		info.Type = "array"
-		info.Alloc = len(c.array) * 4
+		info.Alloc = len(c.array) * 4 // sizeof(uint32)
 	} else if c.isRun() {
 		info.Type = "run"
-		info.Alloc = len(c.runs)*8 + 2
+		info.Alloc = len(c.runs)*interval32Size + runCountHeaderSize
 	} else {
 		info.Type = "bitmap"
-		info.Alloc = len(c.bitmap) * 8
+		info.Alloc = len(c.bitmap) * 8 // sizeof(uint64)
 	}
 
 	if c.mapped {
@@ -2233,6 +2247,7 @@ func unionBitmapRun(a, b *container) *container {
 const maxBitmap = 0xFFFFFFFFFFFFFFFF
 
 // sets all bits in [i, j] (inclusive) (c must be a bitmap container)
+// TODO inclusive upper limit is inconsistent with other functions (CountRange, SliceRange, ForEachRange, OffsetRange(?))
 func (c *container) bitmapSetRange(i, j uint64) {
 	j += 1
 	x := i / 64
@@ -2257,6 +2272,7 @@ func (c *container) bitmapSetRange(i, j uint64) {
 }
 
 // xor's all bits in [i, j] with all true (inclusive) (c must be a bitmap container).
+// TODO inclusive upper limit is inconsistent with other functions
 func (c *container) bitmapXorRange(i, j uint64) {
 	j += 1
 	x := i / 64
@@ -2283,6 +2299,7 @@ func (c *container) bitmapXorRange(i, j uint64) {
 }
 
 // zeroes all bits in [i, j] (inclusive) (c must be a bitmap container)
+// TODO inclusive upper limit is inconsistent with other functions
 func (c *container) bitmapZeroRange(i, j uint64) {
 	j += 1
 	x := i / 64
@@ -2486,8 +2503,6 @@ func differenceRunBitmap(a, b *container) *container {
 }
 
 func differenceRunRun(a, b *container) *container {
-	// (rc *runContainer32) AndNotRunContainer32(b *runContainer32) *runContainer32 {
-
 	if a.n == 0 || b.n == 0 {
 		return a.clone()
 	}

From 970795bac12164cd4978e3cc2729a22b787cea6c Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Tue, 13 Jun 2017 17:44:07 -0500
Subject: [PATCH 046/100] Add new architecture docs page with roaring storage
 section

---
 docs/architecture.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 docs/architecture.md

diff --git a/docs/architecture.md b/docs/architecture.md
new file mode 100644
index 000000000..69912cca5
--- /dev/null
+++ b/docs/architecture.md
@@ -0,0 +1,21 @@
++++
+title = "Architecture"
++++
+
+## Architecture
+
+### Roaring bitmap storage format
+
+Bitmaps are persisted to disk using a file format very similar to the [Roaring Bitmap format spec](https://github.com/RoaringBitmap/RoaringFormatSpec). Pilosa's format uses 64-bit IDs, so it is not binary-compatible with the spec. Some parts of the format are simpler, and an additional section is included. Specific differences include:
+
+* The cookie is always bytes 0-3; the container count is always bytes 4-7, never bytes 2-3.
+* The cookie includes file format version in bytes 2-3 (currently equal to zero).
+* The offset header section is always included.
+* RLE runs are serialized as [start, last], not [start, length].
+* After the container storage section is an operation log, of unspecified length.
+
+![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.svg)
+
+All values are little-endian. The first two bytes of the cookie is 12346 when the file contains no RLE containers, or 12347 when it does. In the no-RLE case, the runFlagBitset is absent. Otherwise the format is identical in both cases. Container types are determined by their cardinality - a container with 4096 or more values is a bitmap, a container with fewer is an array or RLE container. A high bit in runFlagBitset indicates an RLE container.
+
+Storing the runFlagBitset in a separate section, indicated by the cookie value, keeps this format backward compatible with older storage versions that do not support RLE containers.

From ccef8be8440ffbf3b2627608d77851bcef503758 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Wed, 14 Jun 2017 13:52:57 -0500
Subject: [PATCH 047/100] added difference for run and array containers

---
 roaring/roaring.go               | 55 +++++++++++++++++++++++++++++++-
 roaring/roaring_internal_test.go | 22 +++++++++++--
 2 files changed, 73 insertions(+), 4 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index a752670c3..5fc61f835 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2394,6 +2394,7 @@ func difference(a, b *container) *container {
 	}
 }
 
+// differenceArrayArray computes the difference bween two arrays.
 func differenceArrayArray(a, b *container) *container {
 	output := &container{}
 	na, nb := len(a.array), len(b.array)
@@ -2418,6 +2419,7 @@ func differenceArrayArray(a, b *container) *container {
 	return output
 }
 
+// differenceArrayRun computes the difference of an array from a run.
 func differenceArrayRun(a, b *container) *container {
 	// func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
 
@@ -2458,6 +2460,7 @@ func differenceArrayRun(a, b *container) *container {
 	return output
 }
 
+// differenceBitmapRun computes the difference of an bitmap from a run.
 func differenceBitmapRun(a, b *container) *container {
 	if a.n == 0 || b.n == 0 {
 		return a.clone()
@@ -2470,16 +2473,65 @@ func differenceBitmapRun(a, b *container) *container {
 	return output
 }
 
+// differenceRunArray computes the difference of an run from a array.
 func differenceRunArray(a, b *container) *container {
-	// TODO
 	if a.n == 0 || b.n == 0 {
 		return a.clone()
 	}
 
 	output := &container{runs: make([]interval32, 0, a.n)}
+	i := 0
+	j := 0
+	vr := a.runs[j]
+	working := true
+	for working {
+		switch {
+		case b.array[i] < vr.start: //before
+		case b.array[i] > vr.last: //after
+			if vr.start <= vr.last {
+				output.n += output.runAppendInterval(vr)
+			}
+			j++
+			if j < len(a.runs) {
+				vr = a.runs[j]
+			} else {
+				working = false
+			}
+		case b.array[i] == vr.start: //begining of run
+			vr.start++
+		case b.array[i] == a.runs[j].last: //end of run
+			vr.last--
+			if vr.last >= vr.start {
+				output.n += output.runAppendInterval(vr)
+			}
+			j++
+			if j < len(a.runs) {
+				vr = a.runs[j]
+			} else {
+				working = false
+			}
+		case b.array[i] > vr.start: //inside run
+			output.n += output.runAppendInterval(interval32{start: vr.start, last: b.array[i] - 1})
+			vr.start = b.array[i] + 1
+
+		}
+		i++
+		if i >= len(b.array) {
+			working = false
+		}
+	}
+	if vr.start <= vr.last {
+		output.n += output.runAppendInterval(vr)
+	}
+	if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
+		output.runToArray()
+	} else if len(output.runs) > RunMaxSize {
+		output.runToBitmap()
+	}
 	return output
 }
 
+// differenceRunBitmap computes the difference of an run from a bitmap.
 func differenceRunBitmap(a, b *container) *container {
 	// TODO
 	if a.n == 0 || b.n == 0 {
@@ -2502,6 +2554,7 @@ func differenceRunBitmap(a, b *container) *container {
 	return output
 }
 
+// differenceRunRun computes the difference of two runs.
 func differenceRunRun(a, b *container) *container {
 	if a.n == 0 || b.n == 0 {
 		return a.clone()
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index c3d0c5ac8..ae1e2ccce 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1113,7 +1113,6 @@ func TestDifferenceArrayRun(t *testing.T) {
 	}
 }
 
-/*
 func TestDifferenceRunArray(t *testing.T) {
 	a := &container{}
 	b := &container{}
@@ -1127,6 +1126,21 @@ func TestDifferenceRunArray(t *testing.T) {
 			array: []uint32{5, 6, 7, 8, 9, 10},
 			exp:   []interval32{{start: 0, last: 4}, {start: 11, last: 12}},
 		},
+		{
+			runs:  []interval32{{start: 0, last: 12}},
+			array: []uint32{0, 1, 2, 3},
+			exp:   []interval32{{start: 4, last: 12}},
+		},
+		{
+			runs:  []interval32{{start: 0, last: 12}},
+			array: []uint32{9, 10, 11, 12, 13},
+			exp:   []interval32{{start: 0, last: 8}},
+		},
+		{
+			runs:  []interval32{{start: 1, last: 12}},
+			array: []uint32{0, 9, 10, 11, 12, 13},
+			exp:   []interval32{{start: 1, last: 8}},
+		},
 	}
 	for i, test := range tests {
 		a.runs = test.runs
@@ -1134,12 +1148,14 @@ func TestDifferenceRunArray(t *testing.T) {
 		b.array = test.array
 		b.n = len(b.array)
 		ret := differenceRunArray(a, b)
-		if !reflect.DeepEqual(ret.array, test.exp) {
-			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array)
+		if !reflect.DeepEqual(ret.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
 		}
 	}
 }
 
+/*
+
 func TestDifferenceRunBitmap(t *testing.T) {
 	a := &container{}
 	b := &container{bitmap: make([]uint64, bitmapN)}

From 6609120464496669fd54d01cf885088274c71124 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Wed, 14 Jun 2017 16:48:54 -0500
Subject: [PATCH 048/100] replaced linear search with binary search in run
 container

---
 roaring/roaring.go      | 26 +++++++++++++++-----------
 roaring/roaring_test.go |  1 -
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 5fc61f835..bff94ad26 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1337,18 +1337,22 @@ func (c *container) bitmapContains(v uint32) bool {
 	return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0
 }
 
-func (c *container) runContains(v uint32) bool {
-	// TODO binary search
-	for _, iv := range c.runs {
-		if v > iv.last {
-			continue
-		}
-		if v < iv.start {
-			return false
-		}
-		return true
+//runBinSearch uses a binary search of the runs and returns the index of nearest
+//run.
+func runBinSearch(v uint32, a []interval32) (int, bool) {
+	i := sort.Search(len(a),
+		func(i int) bool { return a[i].last >= v })
+	if i < len(a) {
+		return i, (v >= a[i].start) && (v <= a[i].last)
 	}
-	return false
+
+	return i, false
+}
+
+//runContains determines if v is in the containers run set.
+func (c *container) runContains(v uint32) bool {
+	_, found := runBinSearch(v, c.runs)
+	return found
 }
 
 // remove removes a value from the container.
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 07fd99d84..40f6c515d 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -360,7 +360,6 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) {
 		}
 
 		// Remove all values in random order.
-		fmt.Println(len(a))
 		for _, i := range rand.Perm(len(a)) {
 			removed, _ := bm.Remove(a[i])
 			if removed {

From 8e10e27cfac1fbdc679f9513ca24bd2776f3fdf9 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 14 Jun 2017 16:59:20 -0500
Subject: [PATCH 049/100] Remove debugging code

---
 fragment_test.go | 38 --------------------------------------
 1 file changed, 38 deletions(-)

diff --git a/fragment_test.go b/fragment_test.go
index 13cc4e808..38c3ed83b 100644
--- a/fragment_test.go
+++ b/fragment_test.go
@@ -272,44 +272,6 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
 	}
 }
 
-// temporary test for catching op log bug
-func TestFragment_TopN_Intersect_Moderate(t *testing.T) {
-	f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
-	defer f.Close()
-
-	// Create an intersecting input row.
-	src := pilosa.NewBitmap(
-		80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
-		90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
-	)
-
-	// Set bits on rows 0 - 99. Higher rows have higher bit counts.
-	for i := uint64(0); i < 100; i++ {
-		for j := uint64(0); j < i; j++ {
-			f.MustSetBits(i, j)
-		}
-	}
-	f.RecalculateCache()
-
-	// Retrieve top rows.
-	if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil {
-		t.Fatal(err)
-	} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
-		{ID: 99, Count: 19},
-		{ID: 98, Count: 18},
-		{ID: 97, Count: 17},
-		{ID: 96, Count: 16},
-		{ID: 95, Count: 15},
-		{ID: 94, Count: 14},
-		{ID: 93, Count: 13},
-		{ID: 92, Count: 12},
-		{ID: 91, Count: 11},
-		{ID: 90, Count: 10},
-	}) {
-		t.Fatalf("unexpected pairs: %s", spew.Sdump(pairs))
-	}
-}
-
 // Ensure a fragment can return top rows when specified by ID.
 func TestFragment_TopN_IDs(t *testing.T) {
 	f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)

From 76a05d38b511988eb70b3e5b365f5cae8aad5e9a Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 14 Jun 2017 17:00:06 -0500
Subject: [PATCH 050/100] Add test TODO comments

---
 roaring/roaring_test.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 40f6c515d..c14feee52 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -55,6 +55,7 @@ func TestBitmap_Slice_Empty(t *testing.T) {
 }
 
 // Ensure a bitmap can return a slice of values within a range.
+// TODO duplicate for all container types
 func TestBitmap_SliceRange(t *testing.T) {
 	if a := roaring.NewBitmap(0, 1000001, 1000002, 1000003).SliceRange(1, 1000003); !reflect.DeepEqual(a, []uint64{1000001, 1000002}) {
 		t.Fatalf("unexpected slice: %+v", a)
@@ -384,6 +385,7 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) {
 	})
 }
 
+
 func TestBitmap_Marshal_Quick_Array1(t *testing.T)  { testBitmapMarshalQuick(t, 1000, 1000, 2000, false) }
 func TestBitmap_Marshal_Quick_Array2(t *testing.T)  { testBitmapMarshalQuick(t, 10000, 0, 1000, false) }
 func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 10000, false) }
@@ -398,6 +400,8 @@ func TestBitmap_Marshal_Quick_Bitmap_Sorted(t *testing.T) {
 	testBitmapMarshalQuick(t, 10000, 0, 10000, true)
 }
 
+// TODO update for RLE
+
 // Ensure a bitmap can be marshaled and unmarshaled.
 func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
 	if testing.Short() {
@@ -461,6 +465,7 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
 }
 
 // Ensure iterator can iterate over all the values on the bitmap.
+// TODO duplicate for all container types
 func TestIterator(t *testing.T) {
 	itr := roaring.NewBitmap(1, 2, 3).Iterator()
 	itr.Seek(0)

From 18aacbb65a3d129ffe5399386766365ac3905907 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 14 Jun 2017 17:07:08 -0500
Subject: [PATCH 051/100] Move String functions to test file

---
 roaring/roaring.go               |  5 -----
 roaring/roaring_internal_test.go | 10 ++++++++++
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index bff94ad26..ec491f3b3 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1046,11 +1046,6 @@ func (iv interval32) runlen() int {
 	return 1 + int(iv.last) - int(iv.start)
 }
 
-// String produces a human viewable string of the contents.
-func (iv interval32) String() string {
-	return fmt.Sprintf("[%d, %d]", iv.start, iv.last)
-}
-
 // newContainer returns a new instance of container.
 func newContainer() *container {
 	return &container{}
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index ae1e2ccce..dda7e2379 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -15,11 +15,21 @@
 package roaring
 
 import (
+	"fmt"
 	"bytes"
 	"reflect"
 	"testing"
 )
 
+// String produces a human viewable string of the contents.
+func (iv interval32) String() string {
+	return fmt.Sprintf("[%d, %d]", iv.start, iv.last)
+}
+
+func (c *container) String() string {
+	return fmt.Sprintf("<%s container (%v), n=%d, array[%d], runs[%d], bitmap[%d]>", c.info().Type, &c, c.n, len(c.array), len(c.runs), len(c.bitmap))
+}
+
 func TestRunAppendInterval(t *testing.T) {
 	a := container{}
 	tests := []struct {

From b7a790e320f5158ed8e123d213f333de7a94fb01 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 14 Jun 2017 17:09:07 -0500
Subject: [PATCH 052/100] Add iterator tests and WIP seek() for runs

---
 roaring/roaring.go               |  16 +++-
 roaring/roaring_internal_test.go | 154 +++++++++++++++++++++++++++++++
 2 files changed, 166 insertions(+), 4 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index ec491f3b3..d6a52acc3 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -864,11 +864,19 @@ func (itr *Iterator) Seek(seek uint64) {
 	}
 
 	if c.isRun() {
-		// TODO work for seek!=0
-		itr.i, itr.j, itr.k = 0, 0, -1
-		if seek != 0 {
-			panic("cant seeek nonzero")
+		if seek == 0 {
+			itr.i, itr.j, itr.k = 0, 0, -1
 		}
+
+		j, contains := runBinSearch(lb, c.runs)
+		if !contains {
+			itr.j = -1
+			itr.k = -1
+		} else {
+			itr.j = j
+			itr.k = int(lb) - int(c.runs[j].start) - 1
+		}
+
 		return
 	}
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index dda7e2379..62973bf00 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1484,3 +1484,157 @@ func TestXorBitmapRun(t *testing.T) {
 	}
 
 }
+
+func TestIteratorArray(t *testing.T) {
+	// use values that span two containers
+	b := NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000)
+	if !b.containers[0].isArray() {
+		t.Fatalf("wrong container type")
+	}
+
+	itr := b.Iterator()
+	if !(itr.i == 0 && itr.j == -1) {
+		t.Fatalf("iterator did not zero correctly: %v\n", itr)
+	}
+
+	itr.Seek(1000)
+	if !(itr.i == 0 && itr.j == 3) {
+		t.Fatalf("iterator did not seek correctly: %v\n", itr)
+	}
+
+	itr.Seek(10000)
+	itr.Next()
+	val, eof := itr.Next()
+	if !(itr.i == 1 && itr.j == 0 && val == 90000 && !eof) {
+		t.Fatalf("iterator did not next correctly across containers: %v\n", itr)
+	}
+
+	itr.Seek(80000)
+	if !(itr.i == 1 && itr.j == -1) {
+		t.Fatalf("iterator did not seek missing value correctly: %v\n", itr)
+	}
+
+	itr.Seek(100000)
+	if !(itr.i == 1 && itr.j == 0) {
+		t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr)
+	}
+
+	val, eof = itr.Next()
+	if !(val == 100000 && !eof) {
+		t.Fatalf("iterator did not next correctly: %d, %v\n", val, eof)
+	}
+
+	val, eof = itr.Next()
+	if !(val == 0 && eof) {
+		t.Fatalf("iterator did not eof correctly: %d, %v\n", val, eof)
+	}
+}
+
+func TestIteratorBitmap(t *testing.T) {
+	// use values that span two containers
+	// this dataset will update to bitmap after enough Adds,
+	// but won't update to RLE until Optimize() is called
+	b := NewBitmap()
+	for i := uint64(61000); i<71000; i++ {
+		b.Add(i)
+	}
+	for i := uint64(75000); i<75100; i++ {
+		b.Add(i)
+	}
+	if !b.containers[0].isBitmap() {
+		t.Fatalf("wrong container type")
+	}
+
+	itr := b.Iterator()
+	if !(itr.i == 0 && itr.j == -1) {
+		t.Fatalf("iterator did not zero correctly: %v\n", itr)
+	}
+
+	itr.Seek(65000)
+	if !(itr.i == 0 && itr.j == 64999) {
+		t.Fatalf("iterator did not seek correctly: %v\n", itr)
+	}
+
+	itr.Seek(65535)
+	itr.Next()
+	val, eof := itr.Next()
+	if !(itr.i == 1 && itr.j == 0 && val == 65536 && !eof) {
+		t.Fatalf("iterator did not next correctly across containers: %v\n", itr)
+	}
+
+	itr.Seek(74000)
+	if !(itr.i == 1 && itr.j == 8463) {
+		t.Fatalf("iterator did not seek missing value correctly: %v\n", itr)
+	}
+
+	itr.Seek(70999)
+	if !(itr.i == 1 && itr.j == 5462) {
+		t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr)
+	}
+
+	val, eof = itr.Next()
+	if !(val == 70999 && !eof) {
+		t.Fatalf("iterator did not next correctly: %d, %v\n", val, eof)
+	}
+
+	itr.Seek(75100)
+	val, eof = itr.Next()
+	if !(val == 0 && eof) {
+		t.Fatalf("iterator did not eof correctly: %d, %v\n", val, eof)
+	}
+}
+
+func TestIteratorRuns(t *testing.T) {
+	b := NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005)
+	b.Optimize()
+	if !b.containers[0].isRun() {
+		t.Fatalf("wrong container type")
+	}
+
+	itr := b.Iterator()
+	if !(itr.i == 0 && itr.j == 0 && itr.k == -1) {
+		t.Fatalf("iterator did not zero correctly: %v\n", itr)
+	}
+
+	itr.Seek(4)
+	if !(itr.i == 0 && itr.j == 0 && itr.k == 3) {
+		t.Fatalf("iterator did not seek correctly: %v\n", itr)
+	}
+	itr.Next()
+	val, eof := itr.Next()
+	if !(val == 1000 && !eof) {
+		t.Fatalf("iterator did not next correctly across runs")
+	}
+	itr.Next()
+	val, eof = itr.Next()
+	if !(val == 1002 && !eof) {
+		t.Fatalf("iterator did not next correctly within a run")
+	}
+	itr.Next()
+	itr.Next()
+	itr.Next()
+	val, eof = itr.Next()
+	if !(val == 100000 && !eof) {
+		t.Fatalf("iterator did not next correctly across containers")
+	}
+
+	itr.Seek(500)
+	if !(itr.i == 0 && itr.j == 1 && itr.k == -1) {
+		t.Fatalf("iterator did not seek missing value correctly: %v\n", itr)
+	}
+
+	itr.Seek(1004)
+	if !(itr.i == 0 && itr.j == 1 && itr.k == 3) {
+		t.Fatalf("iterator did not seek correctly in multiple runs: %v\n", itr)
+	}
+
+	itr.Seek(100005)
+	if !(itr.i == 1 && itr.j == 0 && itr.k == 4) {
+		t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr)
+	}
+
+	val, eof = itr.Next()
+	if !(val == 0 && eof) {
+		t.Fatalf("iterator did not eof correctly: %d, %v\n", val, eof)
+	}
+}

From 056670389ef538f206160cf96e43256eca717320 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Wed, 14 Jun 2017 17:13:15 -0500
Subject: [PATCH 053/100] tests for binSearchRuns

---
 roaring/roaring.go               |  4 +-
 roaring/roaring_internal_test.go | 66 +++++++++++++++++++++++++++++---
 2 files changed, 62 insertions(+), 8 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index d6a52acc3..2de26d812 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1342,7 +1342,7 @@ func (c *container) bitmapContains(v uint32) bool {
 
 //runBinSearch uses a binary search of the runs and returns the index of nearest
 //run.
-func runBinSearch(v uint32, a []interval32) (int, bool) {
+func binSearchRuns(v uint32, a []interval32) (int, bool) {
 	i := sort.Search(len(a),
 		func(i int) bool { return a[i].last >= v })
 	if i < len(a) {
@@ -1354,7 +1354,7 @@ func runBinSearch(v uint32, a []interval32) (int, bool) {
 
 //runContains determines if v is in the containers run set.
 func (c *container) runContains(v uint32) bool {
-	_, found := runBinSearch(v, c.runs)
+	_, found := binSearchRuns(v, c.runs)
 	return found
 }
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 62973bf00..400f99b66 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -15,8 +15,8 @@
 package roaring
 
 import (
-	"fmt"
 	"bytes"
+	"fmt"
 	"reflect"
 	"testing"
 )
@@ -1468,9 +1468,10 @@ func TestXorBitmapRun(t *testing.T) {
 		runs   []interval32
 		exp    []uint64
 	}{
-		{bitmap: []uint64{0x0, 0x0, 0x0},
-			runs: []interval32{{start: 129, last: 131}},
-			exp:  []uint64{0x0, 0x0, 0x00000000000000E},
+		{
+			bitmap: []uint64{0x0, 0x0, 0x0},
+			runs:   []interval32{{start: 129, last: 131}},
+			exp:    []uint64{0x0, 0x0, 0x00000000000000E},
 		},
 	}
 	for i, test := range tests {
@@ -1535,10 +1536,10 @@ func TestIteratorBitmap(t *testing.T) {
 	// this dataset will update to bitmap after enough Adds,
 	// but won't update to RLE until Optimize() is called
 	b := NewBitmap()
-	for i := uint64(61000); i<71000; i++ {
+	for i := uint64(61000); i < 71000; i++ {
 		b.Add(i)
 	}
-	for i := uint64(75000); i<75100; i++ {
+	for i := uint64(75000); i < 75100; i++ {
 		b.Add(i)
 	}
 	if !b.containers[0].isBitmap() {
@@ -1638,3 +1639,56 @@ func TestIteratorRuns(t *testing.T) {
 		t.Fatalf("iterator did not eof correctly: %d, %v\n", val, eof)
 	}
 }
+
+func TestRunBinSearchContains(t *testing.T) {
+	tests := []struct {
+		runs  []interval32
+		index uint32
+		exp   struct {
+			index int
+			found bool
+		}
+	}{
+		{
+			runs:  []interval32{{start: 0, last: 10}},
+			index: uint32(3),
+			exp: struct {
+				index int
+				found bool
+			}{index: 0, found: true},
+		},
+		{
+			runs:  []interval32{{start: 0, last: 10}},
+			index: uint32(13),
+			exp: struct {
+				index int
+				found bool
+			}{index: 0, found: false},
+		},
+		{
+			runs:  []interval32{{start: 0, last: 10}, {start: 20, last: 30}},
+			index: uint32(13),
+			exp: struct {
+				index int
+				found bool
+			}{index: 0, found: false},
+		},
+		{
+			runs:  []interval32{{start: 0, last: 10}, {start: 20, last: 30}},
+			index: uint32(36),
+			exp: struct {
+				index int
+				found bool
+			}{index: 1, found: false},
+		},
+	}
+	for i, test := range tests {
+		index := test.index
+		runs := test.runs
+		idx, found := binSearchRuns(index, runs)
+
+		if test.exp.index != idx && test.exp.found != found {
+			t.Fatalf("test #%v expected %v , but got %v %v", i, test.exp, idx, found)
+		}
+	}
+}

From d0ee8a6173375830e1e14ebef2d34d4c8b32c816 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 14 Jun 2017 17:38:50 -0500
Subject: [PATCH 054/100] Add test and comment for runBinSearch

---
 roaring/roaring.go               |  6 +--
 roaring/roaring_internal_test.go | 64 ++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 3 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 2de26d812..3d666056d 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1340,9 +1340,9 @@ func (c *container) bitmapContains(v uint32) bool {
 	return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0
 }
 
-//runBinSearch uses a binary search of the runs and returns the index of nearest
-//run.
-func binSearchRuns(v uint32, a []interval32) (int, bool) {
+// runBinSearch returns the index of the run containing v, and true, when v is contained;
+// or the index of the next run starting after v, and false, when v is not contained.
+func runBinSearch(v uint32, a []interval32) (int, bool) {
 	i := sort.Search(len(a),
 		func(i int) bool { return a[i].last >= v })
 	if i < len(a) {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 400f99b66..0c4bf8e0d 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1692,3 +1692,67 @@ func TestRunBinSearchContains(t *testing.T) {
 		}
 	}
 }
+
+func TestRunBinSearch(t *testing.T) {
+	tests := []struct {
+		runs   []interval32
+		search uint32
+		exp    bool
+		expi   int
+	}{
+		{
+			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			search: 1,
+			exp:    false,
+			expi:   0,
+		},
+		{
+			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			search: 2,
+			exp:    true,
+			expi:   0,
+		},
+		{
+			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			search: 5,
+			exp:    true,
+			expi:   0,
+		},
+		{
+			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			search: 10,
+			exp:    true,
+			expi:   0,
+		},
+		{
+			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			search: 20,
+			exp:    false,
+			expi:   1,
+		},
+		{
+			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			search: 55,
+			exp:    true,
+			expi:   1,
+		},
+		{
+			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			search: 70,
+			exp:    false,
+			expi:   2,
+		},
+		{
+			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			search: 100,
+			exp:    false,
+			expi:   3,
+		},
+	}
+	for i, test := range tests {
+		idx, contains := runBinSearch(test.search, test.runs)
+		if !(test.exp == contains && test.expi == idx) {
+			t.Fatalf("test #%v expected (%v, %v) but got (%v, %v)", i, test.exp, test.expi, contains, idx)
+		}
+	}
+}

From 8ed9005b17a9374a96db0a7b45ce13ec436d874a Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 14 Jun 2017 17:45:54 -0500
Subject: [PATCH 055/100] Update name of search function

---
 roaring/roaring.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 3d666056d..f5bda8cf6 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -868,7 +868,7 @@ func (itr *Iterator) Seek(seek uint64) {
 			itr.i, itr.j, itr.k = 0, 0, -1
 		}
 
-		j, contains := runBinSearch(lb, c.runs)
+		j, contains := binSearchRuns(lb, c.runs)
 		if !contains {
 			itr.j = -1
 			itr.k = -1

From a0d98cfb30c48f121751885b93d0d1c0aae2011f Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Thu, 15 Jun 2017 11:13:34 -0500
Subject: [PATCH 056/100] support for difference of run and bitmap

---
 roaring/roaring.go               | 55 ++++++++++++++++++++++++++------
 roaring/roaring_internal_test.go | 28 ++++++++++++++--
 2 files changed, 71 insertions(+), 12 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index f5bda8cf6..354bda36e 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2540,7 +2540,6 @@ func differenceRunArray(a, b *container) *container {
 
 // differenceRunBitmap computes the difference of an run from a bitmap.
 func differenceRunBitmap(a, b *container) *container {
-	// TODO
 	if a.n == 0 || b.n == 0 {
 		return a.clone()
 	}
@@ -2548,16 +2547,54 @@ func differenceRunBitmap(a, b *container) *container {
 	output := &container{runs: make([]interval32, 0, a.n)}
 	itr := newBufIterator(newBitmapIterator(b.bitmap))
 
-	for i := 0; ; {
-		vb, eof := itr.next()
-		if eof {
-			break
+	vb, eof := itr.next()
+	j := 0
+	vr := a.runs[j]
+	working := !eof
+	for working {
+		switch {
+		case vb < vr.start: //before
+		case vb > vr.last: //after
+			if vr.start <= vr.last {
+				output.n += output.runAppendInterval(vr)
+			}
+			j++
+			if j < len(a.runs) {
+				vr = a.runs[j]
+			} else {
+				working = false
+			}
+		case vb == vr.start: //begining of run
+			vr.start++
+		case vb == a.runs[j].last: //end of run
+			vr.last--
+			if vr.last >= vr.start {
+				output.n += output.runAppendInterval(vr)
+			}
+			j++
+			if j < len(a.runs) {
+				vr = a.runs[j]
+			} else {
+				working = false
+			}
+		case vb > vr.start: //inside run
+			output.n += output.runAppendInterval(interval32{start: vr.start, last: vb - 1})
+			vr.start = vb + 1
+
+		}
+		vb, eof = itr.next()
+		if eof {
+			working = false
 		}
-		fmt.Println(i, vb)
-
-		i++
 	}
-
+	if vr.start <= vr.last {
+		output.n += output.runAppendInterval(vr)
+	}
+	if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
+		output.runToArray()
+	} else if len(output.runs) > RunMaxSize {
+		output.runToBitmap()
+	}
 	return output
 }
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 0c4bf8e0d..29b95f52d 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1164,8 +1164,6 @@ func TestDifferenceRunArray(t *testing.T) {
 	}
 }
 
-/*
-
 func TestDifferenceRunBitmap(t *testing.T) {
 	a := &container{}
 	b := &container{bitmap: make([]uint64, bitmapN)}
@@ -1179,6 +1177,31 @@ func TestDifferenceRunBitmap(t *testing.T) {
 			bitmap: []uint64{0x0000FFFF000000F0},
 			exp:    []interval32{{start: 0, last: 3}, {start: 8, last: 31}, {start: 48, last: 63}},
 		},
+		{
+			runs:   []interval32{{start: 0, last: 63}},
+			bitmap: []uint64{0x8000000000000000},
+			exp:    []interval32{{start: 0, last: 62}},
+		},
+		{
+			runs:   []interval32{{start: 0, last: 63}},
+			bitmap: []uint64{0x0000000000000001},
+			exp:    []interval32{{start: 1, last: 63}},
+		},
+		{
+			runs:   []interval32{{start: 0, last: 63}},
+			bitmap: []uint64{0x0, 0x0000000000000001},
+			exp:    []interval32{{start: 0, last: 63}},
+		},
+		{
+			runs:   []interval32{{start: 0, last: 65}},
+			bitmap: []uint64{0x0, 0x0000000000000001},
+			exp:    []interval32{{start: 0, last: 63}, {start: 65, last: 65}},
+		},
+		{
+			runs:   []interval32{{start: 0, last: 65}},
+			bitmap: []uint64{0x0, 0x8000000000000000},
+			exp:    []interval32{{start: 0, last: 65}},
+		},
 	}
 	for i, test := range tests {
 		a.runs = test.runs
@@ -1193,7 +1216,6 @@ func TestDifferenceRunBitmap(t *testing.T) {
 		}
 	}
 }
-*/
 
 func TestDifferenceBitmapRun(t *testing.T) {
 	a := &container{bitmap: make([]uint64, bitmapN)}

From 971d60617b3126d77996e5fa6c39278c203db406 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 15 Jun 2017 11:15:48 -0500
Subject: [PATCH 057/100] Update runRemove with run binary search

---
 roaring/roaring.go               | 37 +++++++++++++++-----------------
 roaring/roaring_internal_test.go |  8 +++----
 2 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 354bda36e..1f5c9be58 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1401,28 +1401,25 @@ func (c *container) bitmapRemove(v uint32) bool {
 	return true
 }
 
+// runRemove removes v from a run container, and returns true if v was removed.
 func (c *container) runRemove(v uint32) bool {
-	// TODO binary search
-	for i, iv := range c.runs {
-		if v <= iv.last {
-			if v < iv.start {
-				return false
-			}
-			c.unmap()
-			if v == iv.last && v == iv.start {
-				c.runs = append(c.runs[:i], c.runs[i+1:]...)
-			} else if v == iv.last {
-				c.runs[i].last -= 1
-			} else if v == iv.start {
-				c.runs[i].start += 1
-			} else if v > iv.start {
-				c.runs[i].last = v - 1
-				c.runs = append(c.runs[:i+1], append([]interval32{{start: v + 1, last: iv.last}}, c.runs[i+1:]...)...)
-			}
-			return true
-		}
+	i, contains := binSearchRuns(v, c.runs)
+	if !contains {
+		return false
 	}
-	return false
+	c.unmap()
+	if v == c.runs[i].last && v == c.runs[i].start {
+		c.runs = append(c.runs[:i], c.runs[i+1:]...)
+	} else if v == c.runs[i].last {
+		c.runs[i].last -= 1
+	} else if v == c.runs[i].start {
+		c.runs[i].start += 1
+	} else if v > c.runs[i].start {
+		last := c.runs[i].last
+		c.runs[i].last = v - 1
+		c.runs = append(c.runs[:i+1], append([]interval32{{start: v+1, last: last}}, c.runs[i+1:]...)...)
+	}
+	return true
 }
 
 // max returns the maximum value in the container.
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 29b95f52d..8e083c9d0 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -295,17 +295,17 @@ func TestRunRemove(t *testing.T) {
 		{44, []interval32{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
 	}
 
-	for _, test := range tests {
+	for i, test := range tests {
 		c.mapped = true
 		ret := c.remove(test.op)
 		if ret != test.expRet || !reflect.DeepEqual(c.runs, test.exp) {
-			t.Fatalf("Unexpected result removing %v from runs. Expected %v, got %v. Expected %v, got %v", test.op, test.expRet, ret, test.exp, c.runs)
+			t.Fatalf("test #%v Unexpected result removing %v from runs. Expected %v, got %v. Expected %v, got %v", i, test.op, test.expRet, ret, test.exp, c.runs)
 		}
 		if ret && c.mapped {
-			t.Fatalf("container was not unmapped although bit %v was removed", test.op)
+			t.Fatalf("test #%v container was not unmapped although bit %v was removed", i, test.op)
 		}
 		if !ret && !c.mapped {
-			t.Fatalf("container was unmapped although bit %v was not removed", test.op)
+			t.Fatalf("test #%v container was unmapped although bit %v was not removed", i, test.op)
 		}
 	}
 }

From 371f36fb76a871228eb75ba59dbad45df55a8a44 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 15 Jun 2017 11:43:28 -0500
Subject: [PATCH 058/100] Add fast check for empty conversions

---
 roaring/roaring.go               | 53 ++++++++++++++++++++++++++++--
 roaring/roaring_internal_test.go | 56 ++++++++++++++++++++++++++++++++
 2 files changed, 106 insertions(+), 3 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 1f5c9be58..39e29de57 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1469,6 +1469,14 @@ func (c *container) runMax() uint32 {
 // bitmapToArray converts from bitmap format to array format.
 func (c *container) bitmapToArray() {
 	c.array = make([]uint32, 0, c.n)
+
+	// return early if empty
+	if c.n == 0 {
+		c.bitmap = nil
+		c.mapped = false
+		return
+	}
+
 	for i, bitmap := range c.bitmap {
 		for bitmap != 0 {
 			t := bitmap & -bitmap
@@ -1483,6 +1491,14 @@ func (c *container) bitmapToArray() {
 // arrayToBitmap converts from array format to bitmap format.
 func (c *container) arrayToBitmap() {
 	c.bitmap = make([]uint64, bitmapN)
+
+	// return early if empty
+	if c.n == 0 {
+		c.array = nil
+		c.mapped = false
+		return
+	}
+
 	for _, v := range c.array {
 		c.bitmap[int(v)/64] |= (uint64(1) << uint(v%64))
 	}
@@ -1493,6 +1509,14 @@ func (c *container) arrayToBitmap() {
 // runToBitmap converts from RLE format to bitmap format.
 func (c *container) runToBitmap() {
 	c.bitmap = make([]uint64, bitmapN)
+
+	// return early if empty
+	if c.n == 0 {
+		c.runs = nil
+		c.mapped = false
+		return
+	}
+
 	for _, r := range c.runs {
 		// TODO is there a faster way ?!?!!
 		for v := r.start; v <= r.last; v++ {
@@ -1505,9 +1529,16 @@ func (c *container) runToBitmap() {
 
 // bitmapToRun converts from bitmap format to RLE format.
 func (c *container) bitmapToRun() {
-	numRuns := c.bitmapCountRuns() // TODO test
+	// return early if empty
+	if c.n == 0 {
+		c.runs = make([]interval32, 0)
+		c.bitmap = nil
+		c.mapped = false
+		return
+	}
+
+	numRuns := c.bitmapCountRuns()
 	c.runs = make([]interval32, 0, numRuns)
-	// TODO return early if no runs
 
 	current := c.bitmap[0]
 	var i, start, last uint32
@@ -1553,7 +1584,15 @@ func (c *container) bitmapToRun() {
 
 // arrayToRun converts from array format to RLE format.
 func (c *container) arrayToRun() {
-	numRuns := c.arrayCountRuns() // TODO test
+	// return early if empty
+	if c.n == 0 {
+		c.runs = make([]interval32, 0)
+		c.array = nil
+		c.mapped = false
+		return
+	}
+
+	numRuns := c.arrayCountRuns()
 	c.runs = make([]interval32, 0, numRuns)
 	start := c.array[0]
 	for i, v := range c.array[1:] {
@@ -1572,6 +1611,14 @@ func (c *container) arrayToRun() {
 // runToArray converts from RLE format to array format.
 func (c *container) runToArray() {
 	c.array = make([]uint32, 0, c.n)
+
+	// return early if empty
+	if c.n == 0 {
+		c.runs = nil
+		c.mapped = false
+		return
+	}
+
 	for _, r := range c.runs {
 		for v := r.start; v <= r.last; v++ {
 			c.array = append(c.array, v)
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 8e083c9d0..62aacf467 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -748,6 +748,10 @@ func TestArrayToBitmap(t *testing.T) {
 		array []uint32
 		exp   []uint64
 	}{
+		{
+			array: []uint32{},
+			exp:   []uint64{},
+		},
 		{
 			array: []uint32{0, 1, 2, 3},
 			exp:   []uint64{0xF},
@@ -761,6 +765,7 @@ func TestArrayToBitmap(t *testing.T) {
 		}
 
 		a.array = test.array
+		a.n = len(test.array)
 		a.arrayToBitmap()
 		if !reflect.DeepEqual(a.bitmap, exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap)
@@ -768,12 +773,47 @@ func TestArrayToBitmap(t *testing.T) {
 	}
 }
 
+func TestBitmapToArray(t *testing.T) {
+	a := &container{}
+	tests := []struct {
+		bitmap []uint64
+		exp    []uint32
+	}{
+		{
+			bitmap: []uint64{},
+			exp:    []uint32{},
+		},
+		{
+			bitmap: []uint64{0xF},
+			exp:    []uint32{0, 1, 2, 3},
+		},
+	}
+	for i, test := range tests {
+		a.bitmap = make([]uint64, bitmapN)
+		n := 0
+		for i, v := range test.bitmap {
+			a.bitmap[i] = v
+			n += int(popcount(v))
+		}
+		a.n = n
+
+		a.bitmapToArray()
+		if !reflect.DeepEqual(a.array, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.array)
+		}
+	}
+}
+
 func TestRunToBitmap(t *testing.T) {
 	a := &container{}
 	tests := []struct {
 		runs []interval32
 		exp  []uint64
 	}{
+		{
+			runs: []interval32{},
+			exp:  []uint64{},
+		},
 		{
 			runs: []interval32{{start: 0, last: 0}},
 			exp:  []uint64{1},
@@ -794,11 +834,14 @@ func TestRunToBitmap(t *testing.T) {
 
 	for i, test := range tests {
 		exp := make([]uint64, bitmapN)
+		n := 0
 		for i, v := range test.exp {
 			exp[i] = v
+			n += int(popcount(v))
 		}
 
 		a.runs = test.runs
+		a.n = n
 		a.runToBitmap()
 		if !reflect.DeepEqual(a.bitmap, exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, exp, a.bitmap)
@@ -812,6 +855,11 @@ func TestBitmapToRun(t *testing.T) {
 		bitmap []uint64
 		exp    []interval32
 	}{
+		{
+			// empty run
+			bitmap: []uint64{},
+			exp:    []interval32{},
+		},
 		{
 			// single-bit run
 			bitmap: []uint64{1},
@@ -870,6 +918,10 @@ func TestArrayToRun(t *testing.T) {
 		array []uint32
 		exp   []interval32
 	}{
+		{
+			array: []uint32{},
+			exp:   []interval32{},
+		},
 		{
 			array: []uint32{0},
 			exp:   []interval32{{start: 0, last: 0}},
@@ -900,6 +952,10 @@ func TestRunToArray(t *testing.T) {
 		runs []interval32
 		exp  []uint32
 	}{
+		{
+			runs: []interval32{},
+			exp:  []uint32{},
+		},
 		{
 			runs: []interval32{{start: 0, last: 0}},
 			exp:  []uint32{0},

From 5f5a8571819aea12ca1344a344dfeab2a5dca8d7 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 15 Jun 2017 11:58:51 -0500
Subject: [PATCH 059/100] Modify signature of bitmap{Set,Zero,Xor}Range to be
 consistent with similar functions

---
 roaring/roaring.go               | 18 ++++++------------
 roaring/roaring_internal_test.go |  6 +++---
 2 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 39e29de57..2c74306bc 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2290,17 +2290,15 @@ func unionBitmapRun(a, b *container) *container {
 	}
 	output := a.clone()
 	for j := 0; j < len(b.runs); j++ {
-		output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last))
+		output.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
 	}
 	return output
 }
 
 const maxBitmap = 0xFFFFFFFFFFFFFFFF
 
-// sets all bits in [i, j] (inclusive) (c must be a bitmap container)
-// TODO inclusive upper limit is inconsistent with other functions (CountRange, SliceRange, ForEachRange, OffsetRange(?))
+// sets all bits in [i, j) (c must be a bitmap container)
 func (c *container) bitmapSetRange(i, j uint64) {
-	j += 1
 	x := i / 64
 	y := (j - 1) / 64
 	var X uint64 = maxBitmap << (i % 64)
@@ -2322,10 +2320,8 @@ func (c *container) bitmapSetRange(i, j uint64) {
 	}
 }
 
-// xor's all bits in [i, j] with all true (inclusive) (c must be a bitmap container).
-// TODO inclusive upper limit is inconsistent with other functions
+// xor's all bits in [i, j) with all true (c must be a bitmap container).
 func (c *container) bitmapXorRange(i, j uint64) {
-	j += 1
 	x := i / 64
 	y := (j - 1) / 64
 	var X uint64 = maxBitmap << (i % 64)
@@ -2349,10 +2345,8 @@ func (c *container) bitmapXorRange(i, j uint64) {
 	}
 }
 
-// zeroes all bits in [i, j] (inclusive) (c must be a bitmap container)
-// TODO inclusive upper limit is inconsistent with other functions
+// zeroes all bits in [i, j) (c must be a bitmap container)
 func (c *container) bitmapZeroRange(i, j uint64) {
-	j += 1
 	x := i / 64
 	y := (j - 1) / 64
 	var X uint64 = maxBitmap << (i % 64)
@@ -2519,7 +2513,7 @@ func differenceBitmapRun(a, b *container) *container {
 
 	output := a.clone()
 	for j := 0; j < len(b.runs); j++ {
-		output.bitmapZeroRange(uint64(b.runs[j].start), uint64(b.runs[j].last))
+		output.bitmapZeroRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
 	}
 	return output
 }
@@ -3343,7 +3337,7 @@ func xorRunRun(a, b *container) *container {
 func xorBitmapRun(a, b *container) *container {
 	output := a.clone()
 	for j := 0; j < len(b.runs); j++ {
-		output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last))
+		output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
 	}
 
 	if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 62aacf467..ecc87c786 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -732,7 +732,7 @@ func TestBitmapSetRange(t *testing.T) {
 			c.bitmap[i] = v
 		}
 		c.n = c.countRange(0, 65535)
-		c.bitmapSetRange(test.start, test.last)
+		c.bitmapSetRange(test.start, test.last+1)
 		if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) {
 			t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)])
 		}
@@ -1010,7 +1010,7 @@ func TestBitmapZeroRange(t *testing.T) {
 			c.bitmap[i] = v
 		}
 		c.n = c.countRange(0, 65535)
-		c.bitmapZeroRange(test.start, test.last)
+		c.bitmapZeroRange(test.start, test.last+1)
 		if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) {
 			t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)])
 		}
@@ -1528,7 +1528,7 @@ func TestBitmapXorRange(t *testing.T) {
 			c.bitmap[i] = v
 		}
 		c.n = c.countRange(0, 65535)
-		c.bitmapXorRange(test.start, test.last)
+		c.bitmapXorRange(test.start, test.last+1)
 		if !reflect.DeepEqual(c.bitmap[:len(test.exp)], test.exp) {
 			t.Fatalf("test %#v expected %x, got %x", i, test.exp, c.bitmap[:len(test.bitmap)])
 		}

From b33bf230c3c3ace3bc48d6c33a752cca4f8158db Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Thu, 15 Jun 2017 12:06:25 -0500
Subject: [PATCH 060/100] refactor difference run array/bitmap to use container
 iterator

---
 roaring/roaring.go | 87 +++++++++++++++++++---------------------------
 1 file changed, 35 insertions(+), 52 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 2c74306bc..233c83cb4 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2523,57 +2523,8 @@ func differenceRunArray(a, b *container) *container {
 	if a.n == 0 || b.n == 0 {
 		return a.clone()
 	}
-
-	output := &container{runs: make([]interval32, 0, a.n)}
-	i := 0
-	j := 0
-	vr := a.runs[j]
-	working := true
-	for working {
-		switch {
-		case b.array[i] < vr.start: //before
-		case b.array[i] > vr.last: //after
-			if vr.start <= vr.last {
-				output.n += output.runAppendInterval(vr)
-			}
-			j++
-			if j < len(a.runs) {
-				vr = a.runs[j]
-			} else {
-				working = false
-			}
-		case b.array[i] == vr.start: //begining of run
-			vr.start++
-		case b.array[i] == a.runs[j].last: //end of run
-			vr.last--
-			if vr.last >= vr.start {
-				output.n += output.runAppendInterval(vr)
-			}
-			j++
-			if j < len(a.runs) {
-				vr = a.runs[j]
-			} else {
-				working = false
-			}
-		case b.array[i] > vr.start: //inside run
-			output.n += output.runAppendInterval(interval32{start: vr.start, last: b.array[i] - 1})
-			vr.start = b.array[i] + 1
-
-		}
-		i++
-		if i >= len(b.array) {
-			working = false
-		}
-	}
-	if vr.start <= vr.last {
-		output.n += output.runAppendInterval(vr)
-	}
-	if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
-		output.runToArray()
-	} else if len(output.runs) > RunMaxSize {
-		output.runToBitmap()
-	}
-	return output
+	itr := newArrayIterator(b.array)
+	return differenceRunIterator(a, itr)
 }
 
 // differenceRunBitmap computes the difference of an run from a bitmap.
@@ -2581,9 +2532,13 @@ func differenceRunBitmap(a, b *container) *container {
 	if a.n == 0 || b.n == 0 {
 		return a.clone()
 	}
+	itr := newBufIterator(newBitmapIterator(b.bitmap))
+	return differenceRunIterator(a, itr)
+}
+
+func differenceRunIterator(a *container, itr containerIterator) *container {
 
 	output := &container{runs: make([]interval32, 0, a.n)}
-	itr := newBufIterator(newBitmapIterator(b.bitmap))
 
 	vb, eof := itr.next()
 	j := 0
@@ -3032,6 +2987,34 @@ func popcount(x uint64) (n uint64) {
 	return x >> 56
 }
 
+// Returns eof as true if there are no values left in the iterator.
+type containerIterator interface {
+	next() (uint32, bool)
+}
+
+// bitmapIterator represents an iterator over container array values.
+type arrayIterator struct {
+	array []uint32
+	i     int
+}
+
+func newArrayIterator(array []uint32) *arrayIterator {
+	return &arrayIterator{
+		array: array,
+		i:     -1,
+	}
+}
+
+// next returns the next value in the array.
+func (itr *arrayIterator) next() (v uint32, eof bool) {
+
+	itr.i++
+	if itr.i >= len(itr.array) {
+		return 0, true
+	}
+	return itr.array[itr.i], false
+}
+
 // bitmapIterator represents an iterator over container bitmap values.
 type bitmapIterator struct {
 	bitmap []uint64

From bfce3a0746a328a6c9d28176fddd382a148ee007 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 15 Jun 2017 14:50:27 -0500
Subject: [PATCH 061/100] Update test, cleanup

---
 roaring/roaring.go               | 7 +++----
 roaring/roaring_internal_test.go | 6 ++++++
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 233c83cb4..711d45994 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1518,7 +1518,7 @@ func (c *container) runToBitmap() {
 	}
 
 	for _, r := range c.runs {
-		// TODO is there a faster way ?!?!!
+		// TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits
 		for v := r.start; v <= r.last; v++ {
 			c.bitmap[int(v)/64] |= (uint64(1) << uint(v%64))
 		}
@@ -1564,10 +1564,9 @@ func (c *container) bitmapToRun() {
 			current = c.bitmap[i]
 		}
 
-		last = 0
 		if current == maxBitmap {
-			last = 64*i + 64 // TODO verify
-			c.runs = append(c.runs, interval32{start, last - 1})
+			// bitmap[1023] == maxBitmap
+			c.runs = append(c.runs, interval32{start, 65535})
 			break
 		}
 		currentLast := uint32(trailingZeroN(^current))
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index ecc87c786..68485bb49 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -895,7 +895,13 @@ func TestBitmapToRun(t *testing.T) {
 			bitmap: []uint64{0xF000000000000000, 0xFFFFFFFFFFFFFFFF, 0xF},
 			exp:    []interval32{{start: 60, last: 131}},
 		},
+		{
+			bitmap: make([]uint64, bitmapN),
+			exp:    []interval32{{start: 65408, last: 65535}},
+		},
 	}
+	tests[8].bitmap[1022] = 0xFFFFFFFFFFFFFFFF
+	tests[8].bitmap[1023] = 0xFFFFFFFFFFFFFFFF
 
 	for i, test := range tests {
 		a.bitmap = make([]uint64, bitmapN)

From ea59de4234fd5a0550f8bb996c8a4c042bc76fc1 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 15 Jun 2017 15:50:24 -0500
Subject: [PATCH 062/100] Update check function for run containers, add tests

---
 roaring/roaring.go      | 15 ++++++++++++---
 roaring/roaring_test.go | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 711d45994..aabd919d0 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1738,15 +1738,24 @@ func (c *container) info() ContainerInfo {
 func (c *container) check() error {
 	var a ErrorList
 
-	if c.n <= ArrayMaxSize {
-		// TODO check run here
+	if c.isArray() {
 		if len(c.array) != c.n {
 			a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(c.array), c.n))
 		}
-	} else {
+	} else if c.isRun() {
+		runCount := c.runCountRange(0, 0xFFFFFFFF)
+		if runCount != c.n {
+			a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", runCount, c.n))
+		}
+	} else if c.isBitmap() {
 		if n := c.bitmapCountRange(0, uint32(len(c.bitmap)*64)); n != c.n {
 			a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.n))
 		}
+	} else {
+		a.Append(fmt.Errorf("empty container"))
+		if c.n != 0 {
+			a.Append(fmt.Errorf("empty container with nonzero count: n=%d", c.n))
+		}
 	}
 
 	if a == nil {
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index c14feee52..580829450 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -28,6 +28,38 @@ import (
 	_ "github.com/pilosa/pilosa/test"
 )
 
+func TestCheckBitmap(t *testing.T) {
+	b := roaring.NewBitmap()
+	for i := uint64(61000); i < 71000; i++ {
+		b.Add(i)
+	}
+	for i := uint64(75000); i < 75100; i++ {
+		b.Add(i)
+	}
+
+	err := b.Check()
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+}
+
+func TestCheckArray(t *testing.T) {
+	b := roaring.NewBitmap(0, 1, 10, 100, 1000, 10000, 90000, 100000)
+	err := b.Check()
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+}
+
+func TestCheckRun(t *testing.T) {
+	b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 100003, 100004, 100005)
+	b.Optimize() // convert to runs
+	err := b.Check()
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+}
+
 // Ensure an empty bitmap returns false if checking for existence.
 func TestBitmap_Contains_Empty(t *testing.T) {
 	if roaring.NewBitmap().Contains(1000) {

From 1ff5aab75346215324c3f3b681c3b16d7379b886 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 15 Jun 2017 16:54:12 -0500
Subject: [PATCH 063/100] Fix Next() for runs

---
 roaring/roaring.go | 29 ++++++++++-------------------
 1 file changed, 10 insertions(+), 19 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index aabd919d0..5fdafb633 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -893,10 +893,10 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
 			return 0, true
 		}
 
-		// Move to the next item in the container if it's an array container.
 		c := itr.bitmap.containers[itr.i]
 		if c.isArray() {
 			if itr.j >= c.n-1 {
+				// Reached end of array, move to the next container.
 				itr.i, itr.j = itr.i+1, -1
 				continue
 			}
@@ -905,30 +905,21 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
 		}
 
 		if c.isRun() {
-			if itr.j >= len(c.runs)-1 {
-				r := c.runs[itr.j]
-				runLength := int(r.last - r.start)
-
-				if itr.k >= runLength {
-					itr.i++
-					itr.j = -1
-					continue
-				} else {
-					itr.k++
-					return itr.peek(), false
-				}
-			}
-
 			r := c.runs[itr.j]
 			runLength := int(r.last - r.start)
 
 			if itr.k >= runLength {
-				itr.k = 0
-				itr.j++
-			} else {
-				itr.k++
+				// Reached end of run, move to the next run.
+				itr.j, itr.k = itr.j+1, -1
 			}
 
+			if itr.j >= len(c.runs) {
+				// Reached end of runs, move to the next container.
+				itr.i, itr.j = itr.i+1, 0
+				continue
+			}
+
+			itr.k++
 			return itr.peek(), false
 		}
 

From d068f53d937c5cb88e2360510fca42890eea7cf4 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 15 Jun 2017 17:05:18 -0500
Subject: [PATCH 064/100] Finish Seek() for runs

---
 roaring/roaring.go               |  9 +++++----
 roaring/roaring_internal_test.go | 13 ++++++++++---
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 5fdafb633..1dcf5e7cb 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -869,12 +869,13 @@ func (itr *Iterator) Seek(seek uint64) {
 		}
 
 		j, contains := binSearchRuns(lb, c.runs)
-		if !contains {
-			itr.j = -1
-			itr.k = -1
-		} else {
+		if contains {
 			itr.j = j
 			itr.k = int(lb) - int(c.runs[j].start) - 1
+		} else {
+			// Set iterator to next value in the Bitmap.
+			itr.j = j
+			itr.k = -1
 		}
 
 		return
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 68485bb49..3ee835d8a 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1686,21 +1686,22 @@ func TestIteratorRuns(t *testing.T) {
 		t.Fatalf("iterator did not seek correctly: %v\n", itr)
 	}
 	itr.Next()
+	itr.Next()
 	val, eof := itr.Next()
 	if !(val == 1000 && !eof) {
-		t.Fatalf("iterator did not next correctly across runs")
+		t.Fatalf("iterator did not next correctly across runs: %v, %v", val, itr)
 	}
 	itr.Next()
 	val, eof = itr.Next()
 	if !(val == 1002 && !eof) {
-		t.Fatalf("iterator did not next correctly within a run")
+		t.Fatalf("iterator did not next correctly within a run: %v, %v", val, itr)
 	}
 	itr.Next()
 	itr.Next()
 	itr.Next()
 	val, eof = itr.Next()
 	if !(val == 100000 && !eof) {
-		t.Fatalf("iterator did not next correctly across containers")
+		t.Fatalf("iterator did not next correctly across containers: %v, %v", val, itr)
 	}
 
 	itr.Seek(500)
@@ -1713,11 +1714,17 @@ func TestIteratorRuns(t *testing.T) {
 		t.Fatalf("iterator did not seek correctly in multiple runs: %v\n", itr)
 	}
 
+	itr.Seek(1005)
+	if !(itr.i == 0 && itr.j == 1 && itr.k == 4) {
+		t.Fatalf("iterator did not seek correctly to end of run: %v\n", itr)
+	}
+
 	itr.Seek(100005)
 	if !(itr.i == 1 && itr.j == 0 && itr.k == 4) {
 		t.Fatalf("iterator did not seek correctly in multiple containers: %v\n", itr)
 	}
 
+	val, eof = itr.Next()
 	val, eof = itr.Next()
 	if !(val == 0 && eof) {
 		t.Fatalf("iterator did not eof correctly: %d, %v\n", val, eof)

From fee3996351737e975002e9f32cb6bc2a706a180a Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Thu, 15 Jun 2017 17:51:28 -0500
Subject: [PATCH 065/100] Fix cookie order and check storage version on read

---
 roaring/roaring.go               | 19 ++++++++++++-------
 roaring/roaring_internal_test.go | 30 ++++++++++++++++++++++++------
 2 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 1dcf5e7cb..7b1d24666 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -35,8 +35,8 @@ const (
 
 	// cookie is the first four bytes in a roaring bitmap file,
 	// formed by joining magicNumber and storageVersion
-	cookieNoRuns = magicNumberNoRuns<<16 + storageVersion
-	cookie       = magicNumber<<16 + storageVersion
+	cookieNoRuns = magicNumberNoRuns + storageVersion<<16
+	cookie       = magicNumber + storageVersion<<16
 
 	// headerBaseSize is the size of the cookie and key count at the beginning of a file.
 	// Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8.
@@ -630,15 +630,20 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		return errors.New("data too small")
 	}
 
-	// Verify the first sizeof(cookie)=4 bytes are a valid cookie.
-	v := binary.LittleEndian.Uint32(data[0:4])
+	// Verify the first two bytes are a valid magicNumber, and second two bytes match current storageVersion.
+	fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
+	fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4]))
 	containsRuns := false
-	if v == cookieNoRuns {
+	if fileMagic == magicNumberNoRuns {
 		// noop
-	} else if v == cookie {
+	} else if fileMagic == magicNumber {
 		containsRuns = true
 	} else {
-		return errors.New("invalid roaring file")
+		return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic)
+	}
+
+	if fileVersion != storageVersion {
+		return fmt.Errorf("wrong roaring version, file is v%d, server requires v%d", fileVersion, storageVersion)
 	}
 
 	// Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)).
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 3ee835d8a..7d35779ce 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1341,8 +1341,14 @@ func TestWriteReadArray(t *testing.T) {
 	ba := &Bitmap{keys: []uint64{0}, containers: []*container{ca}}
 	ba2 := &Bitmap{}
 	var buf bytes.Buffer
-	ba.WriteTo(&buf)
-	ba2.UnmarshalBinary(buf.Bytes())
+	_, err := ba.WriteTo(&buf)
+	if err != nil {
+		t.Fatalf("error writing: %v", err)
+	}
+	err = ba2.UnmarshalBinary(buf.Bytes())
+	if err != nil {
+		t.Fatalf("error unmarshaling: %v", err)
+	}
 	if !reflect.DeepEqual(ba2.containers[0].array, ca.array) {
 		t.Fatalf("array test expected %x, but got %x", ca.array, ba2.containers[0].array)
 	}
@@ -1357,8 +1363,14 @@ func TestWriteReadBitmap(t *testing.T) {
 	bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}}
 	bb2 := &Bitmap{}
 	var buf bytes.Buffer
-	bb.WriteTo(&buf)
-	bb2.UnmarshalBinary(buf.Bytes())
+	_, err := bb.WriteTo(&buf)
+	if err != nil {
+		t.Fatalf("error writing: %v", err)
+	}
+	err = bb2.UnmarshalBinary(buf.Bytes())
+	if err != nil {
+		t.Fatalf("error unmarshaling: %v", err)
+	}
 	if !reflect.DeepEqual(bb2.containers[0].bitmap, cb.bitmap) {
 		t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.containers[0].bitmap)
 	}
@@ -1369,8 +1381,14 @@ func TestWriteReadRun(t *testing.T) {
 	br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}}
 	br2 := &Bitmap{}
 	var buf bytes.Buffer
-	br.WriteTo(&buf)
-	br2.UnmarshalBinary(buf.Bytes())
+	_, err := br.WriteTo(&buf)
+	if err != nil {
+		t.Fatalf("error writing: %v", err)
+	}
+	err = br2.UnmarshalBinary(buf.Bytes())
+	if err != nil {
+		t.Fatalf("error unmarshaling: %v", err)
+	}
 	if !reflect.DeepEqual(br2.containers[0].runs, cr.runs) {
 		t.Fatalf("run test expected %x, but got %x", cr.runs, br2.containers[0].runs)
 	}

From 1b7eef4fc8af064014f994b3c516aede33a6e2e6 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 16 Jun 2017 13:46:58 -0500
Subject: [PATCH 066/100] simplified difference BitmapArray and BitmapBitmap
 and improved coverage

---
 roaring/roaring.go               | 36 +++++++++++++---------
 roaring/roaring_internal_test.go | 52 ++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 14 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 7b1d24666..80ac665f5 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1414,7 +1414,7 @@ func (c *container) runRemove(v uint32) bool {
 	} else if v > c.runs[i].start {
 		last := c.runs[i].last
 		c.runs[i].last = v - 1
-		c.runs = append(c.runs[:i+1], append([]interval32{{start: v+1, last: last}}, c.runs[i+1:]...)...)
+		c.runs = append(c.runs[:i+1], append([]interval32{{start: v + 1, last: last}}, c.runs[i+1:]...)...)
 	}
 	return true
 }
@@ -2687,26 +2687,28 @@ func differenceArrayBitmap(a, b *container) *container {
 func differenceBitmapArray(a, b *container) *container {
 	output := &container{}
 	itr := newBufIterator(newBitmapIterator(a.bitmap))
-	array := b.array
+	i := 0
+	va, eof := itr.next()
 	for {
-		va, eof := itr.next()
 		if eof {
 			break
 		}
 
-		if len(array) == 0 {
+		if i >= len(b.array) {
 			output.add(va)
+			va, eof = itr.next()
 			continue
 		}
 
-		vb := array[0]
+		vb := b.array[i]
 		if va < vb {
 			output.add(va)
+			va, eof = itr.next()
 		} else if va > vb {
-			array = array[1:]
-			itr.unread()
+			i++
 		} else {
-			array = array[1:]
+			i++
+			va, eof = itr.next()
 		}
 	}
 	return output
@@ -2716,19 +2718,25 @@ func differenceBitmapBitmap(a, b *container) *container {
 	output := &container{}
 	itr0 := newBufIterator(newBitmapIterator(a.bitmap))
 	itr1 := newBufIterator(newBitmapIterator(b.bitmap))
+	v0, eof0 := itr0.next()
+	v1, eof1 := itr1.next()
 	for {
-		v0, eof0 := itr0.next()
-		v1, eof1 := itr1.next()
-
 		if eof0 {
 			break
 		} else if eof1 {
 			output.add(v0)
-		} else if v0 < v1 {
+			v0, eof0 = itr0.next()
+			continue
+		}
+		if v0 < v1 {
 			output.add(v0)
-			itr1.unread()
+			v0, eof0 = itr0.next()
 		} else if v0 > v1 {
-			itr0.unread()
+			v1, eof1 = itr1.next()
+		} else {
+			v0, eof0 = itr0.next()
+			v1, eof1 = itr1.next()
+
 		}
 	}
 	return output
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 7d35779ce..77c5af89c 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1307,6 +1307,58 @@ func TestDifferenceBitmapRun(t *testing.T) {
 	}
 }
 
+func TestDifferenceBitmapArray(t *testing.T) {
+	a := &container{bitmap: make([]uint64, bitmapN)}
+	b := &container{}
+	tests := []struct {
+		bitmap []uint64
+		array  []uint32
+		exp    []uint32
+	}{
+		{
+			bitmap: []uint64{0xFF0F},
+			array:  []uint32{0, 1, 2, 3, 4, 5, 6, 7, 10},
+			exp:    []uint32{8, 9, 11, 12, 13, 14, 15},
+		},
+	}
+	for i, test := range tests {
+		a.bitmap = test.bitmap
+		b.array = test.array
+		ret := differenceBitmapArray(a, b)
+		if !reflect.DeepEqual(ret.array, test.exp) {
+			t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.array)
+		}
+	}
+}
+func TestDifferenceBitmapBitmap(t *testing.T) {
+	a := &container{bitmap: make([]uint64, bitmapN)}
+	b := &container{bitmap: make([]uint64, bitmapN)}
+	tests := []struct {
+		abitmap []uint64
+		bbitmap []uint64
+		exp     []uint32
+	}{
+		{
+			abitmap: []uint64{0xFF00FFFFFFFFFFFF},
+			bbitmap: []uint64{0xFFFFFFFFFFFFF000},
+			exp:     []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
+		},
+		{
+			abitmap: []uint64{0xF},
+			bbitmap: []uint64{},
+			exp:     []uint32{0, 1, 2, 3},
+		},
+	}
+	for i, test := range tests {
+		a.bitmap = test.abitmap
+		b.bitmap = test.bbitmap
+		ret := differenceBitmapBitmap(a, b)
+		if !reflect.DeepEqual(ret.array, test.exp) {
+			t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.array)
+		}
+	}
+}
+
 func TestDifferenceRunRun(t *testing.T) {
 	a := &container{}
 	b := &container{}

From f334d122c886858829399f4e43b87aaa85e03e95 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 16 Jun 2017 16:27:28 -0500
Subject: [PATCH 067/100] expand coverage of IntersectRunRun

---
 roaring/roaring_internal_test.go | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 77c5af89c..0c499a795 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -468,17 +468,35 @@ func TestIntersectRunRun(t *testing.T) {
 			exp:   []interval32{{start: 5, last: 5}, {start: 7, last: 10}},
 			expN:  5,
 		},
+		{
+			aruns: []interval32{{start: 20, last: 30}},
+			bruns: []interval32{{start: 5, last: 10}, {start: 19, last: 21}},
+			exp:   []interval32{{start: 20, last: 21}},
+			expN:  2,
+		},
+		{
+			aruns: []interval32{{start: 5, last: 10}},
+			bruns: []interval32{{start: 7, last: 12}},
+			exp:   []interval32{{start: 7, last: 10}},
+			expN:  4,
+		},
+		{
+			aruns: []interval32{{start: 5, last: 12}},
+			bruns: []interval32{{start: 7, last: 10}},
+			exp:   []interval32{{start: 7, last: 10}},
+			expN:  4,
+		},
 	}
 	for i, test := range tests {
 		a.runs = test.aruns
 		b.runs = test.bruns
 		ret := intersectRunRun(a, b)
-		if !reflect.DeepEqual(ret.runs, test.exp) {
-			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
-		}
 		if ret.n != test.expN {
 			t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n)
 		}
+		if !reflect.DeepEqual(ret.runs, test.exp) {
+			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
+		}
 	}
 
 }

From 02cba542e8cebdde0de2850b822cdfd01b2d9858 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 16 Jun 2017 16:27:54 -0500
Subject: [PATCH 068/100] bug in CountRange

---
 roaring/roaring.go      |  2 +-
 roaring/roaring_test.go | 43 +++++++++++++++++++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 80ac665f5..7aeab1e3f 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -196,7 +196,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
 	j := search64(b.keys, highbits(end))
 
 	// If range is entirely in one container then just count that range.
-	if i > 0 && i == j {
+	if i >= 0 && i == j {
 		return uint64(b.containers[i].countRange(lowbits(start), lowbits(end)))
 	}
 
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 580829450..9ab63c748 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -127,6 +127,29 @@ func TestBitmap_Max(t *testing.T) {
 		}
 	}
 }
+func TestBitmap_CountRange(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 2683177)
+	for i := uint64(628); i < 2683301; i++ {
+		bm0.Add(i)
+	}
+	bm0.Add(2683307)
+	if n := bm0.CountRange(1, 2683311); n != 2682674 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
+	if n := bm0.CountRange(2683177, 2683310); n != 125 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
+	if n := bm0.CountRange(2683301, 3000000); n != 1 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
+	if n := bm0.CountRange(0, 1); n != 1 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
+}
 
 func TestBitmap_Intersection(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 2683177)
@@ -140,6 +163,16 @@ func TestBitmap_Intersection(t *testing.T) {
 		t.Fatalf("unexpected n: %d", n)
 	}
 
+}
+func TestBitmap_Intersection_Empty(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 2683177)
+	bm1 := roaring.NewBitmap()
+
+	result := bm0.Intersect(bm1)
+	if n := result.Count(); n != 0 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
 }
 
 func TestBitmap_Difference(t *testing.T) {
@@ -149,11 +182,18 @@ func TestBitmap_Difference(t *testing.T) {
 		bm1.Add(i)
 	}
 	result := bm0.Difference(bm1)
-	//expect to have just 0
 	if n := result.Count(); n != 1 {
 		t.Fatalf("unexpected n: %d", n)
 	}
 }
+func TestBitmap_Difference_Empty(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 2683177)
+	bm1 := roaring.NewBitmap()
+	result := bm0.Difference(bm1)
+	if n := result.Count(); n != 2 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
 
 func TestBitmap_Union(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003)
@@ -417,7 +457,6 @@ func testBitmapQuick(t *testing.T, n int, min, max uint64) {
 	})
 }
 
-
 func TestBitmap_Marshal_Quick_Array1(t *testing.T)  { testBitmapMarshalQuick(t, 1000, 1000, 2000, false) }
 func TestBitmap_Marshal_Quick_Array2(t *testing.T)  { testBitmapMarshalQuick(t, 10000, 0, 1000, false) }
 func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) { testBitmapMarshalQuick(t, 10000, 0, 10000, false) }

From 432e95a4e83f65fd9c4dd3dffec9c3621727dfdc Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 16 Jun 2017 17:30:37 -0500
Subject: [PATCH 069/100] added test for bitmap.clone()

---
 roaring/roaring_test.go | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 9ab63c748..8150f4376 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -28,6 +28,24 @@ import (
 	_ "github.com/pilosa/pilosa/test"
 )
 
+func TestBitmapClone(t *testing.T) {
+	b := roaring.NewBitmap()
+	for i := uint64(61000); i < 71000; i++ {
+		b.Add(i)
+	}
+	c := b.Clone()
+	if !reflect.DeepEqual(b, c) {
+		t.Fatalf("Clone Objects not equal\n")
+	}
+	d := func() *roaring.Bitmap { //anybody know how to declare a nil value?
+		return nil
+	}()
+	e := d.Clone()
+	if e != nil {
+		t.Fatalf("Clone nil Objects not equal\n")
+	}
+}
+
 func TestCheckBitmap(t *testing.T) {
 	b := roaring.NewBitmap()
 	for i := uint64(61000); i < 71000; i++ {
@@ -164,6 +182,22 @@ func TestBitmap_Intersection(t *testing.T) {
 	}
 
 }
+
+func TestBitmap_Union1(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 2683177)
+	bm1 := roaring.NewBitmap()
+	for i := uint64(628); i < 2683301; i++ {
+		bm1.Add(i)
+	}
+	bm1.Add(4000000)
+
+	result := bm0.Union(bm1)
+	if n := result.Count(); n != 2682675 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
+}
+
 func TestBitmap_Intersection_Empty(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 2683177)
 	bm1 := roaring.NewBitmap()

From 4cf56f8b7367cf30b1c2f23f121bd082dfd88690 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Sat, 17 Jun 2017 08:34:33 -0500
Subject: [PATCH 070/100] changed unmarshalling logic to handle larger
 cardinality of runs

---
 roaring/roaring.go | 30 +++++++++++++-----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 7aeab1e3f..958a36d00 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -683,23 +683,19 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 
 		// Map byte slice directly to the container data.
 		c := b.containers[i]
-		if c.n <= ArrayMaxSize {
-			if containsRuns && (runFlagBitset[i/8]&(1<<uint(i%8))) != 0 {
-				// Read runs.
-				runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
-				c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount]
-				opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval32Size
-			} else {
-				// Read array.
-				c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
-				// TODO: instead of commenting this out, we ne ed to make it a configuration option
-				//for _, v := range c.array {
-				//    assert(lowbits(uint64(v)) == v, "array value out of range: %d", v)
-				//}
-				opsOffset = int(offset) + len(c.array)*4 // sizeof(uint32)
-			}
-		} else {
-			// Read bitmap.
+
+		if containsRuns && (runFlagBitset[i/8]&(1<<uint(i%8))) != 0 { // Read runs.
+			runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
+			c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount]
+			opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval32Size
+		} else if c.n <= ArrayMaxSize { // Read array.
+			c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
+			// TODO: instead of commenting this out, we ne ed to make it a configuration option
+			//for _, v := range c.array {
+			//    assert(lowbits(uint64(v)) == v, "array value out of range: %d", v)
+			//}
+			opsOffset = int(offset) + len(c.array)*4 // sizeof(uint32)
+		} else { // Read bitmap.
 			c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
 			opsOffset = int(offset) + len(c.bitmap)*8 // sizeof(uint64)
 		}

From 057be3703e49804e0ae4584b9fc762e2aa1be7e2 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Mon, 19 Jun 2017 11:19:27 -0500
Subject: [PATCH 071/100] a few tweaks after rebasing 334-rle onto master

---
 ctl/check_test.go                | 5 +++--
 roaring/roaring.go               | 4 ++--
 roaring/roaring_internal_test.go | 8 +-------
 3 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/ctl/check_test.go b/ctl/check_test.go
index af223c18d..33a19feb3 100644
--- a/ctl/check_test.go
+++ b/ctl/check_test.go
@@ -17,7 +17,6 @@ package ctl
 import (
 	"bytes"
 	"encoding/hex"
-	"golang.org/x/net/context"
 	"io"
 	"io/ioutil"
 	"math/rand"
@@ -25,6 +24,8 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"context"
 )
 
 func TestCheckCommand_RunCacheFile(t *testing.T) {
@@ -84,7 +85,7 @@ func TestCheckCommand_Run(t *testing.T) {
 	var buf bytes.Buffer
 	io.Copy(&buf, r)
 
-	if err.Error() != "invalid roaring file" {
+	if !strings.HasPrefix(err.Error(), "invalid roaring file") {
 		t.Fatalf("expect error: invalid roaring file, actual: '%s'", err)
 	}
 	//	Todo: need correct roaring file for happy path
diff --git a/roaring/roaring.go b/roaring/roaring.go
index 958a36d00..80dbfa79a 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1333,9 +1333,9 @@ func (c *container) bitmapContains(v uint32) bool {
 	return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0
 }
 
-// runBinSearch returns the index of the run containing v, and true, when v is contained;
+// binSearchRuns returns the index of the run containing v, and true, when v is contained;
 // or the index of the next run starting after v, and false, when v is not contained.
-func runBinSearch(v uint32, a []interval32) (int, bool) {
+func binSearchRuns(v uint32, a []interval32) (int, bool) {
 	i := sort.Search(len(a),
 		func(i int) bool { return a[i].last >= v })
 	if i < len(a) {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 0c499a795..291036ef8 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -223,12 +223,6 @@ func TestBitmapCountRange(t *testing.T) {
 			t.Fatalf("test #%v count of %v from %v to %v should be %v but got %v", i, test.bitmap, test.start, test.end, test.exp, ret)
 		}
 	}
-
-	c = container{bitmap: []uint64{0xF0000001, 0xFF000000, 0xFF00000000000000}}
-	cnt := c.bitmapCountRange(62, 129)
-	if cnt != 10 {
-		t.Fatalf("count of %v from 62 to 129 should be 10, but got %v", c.bitmap, cnt)
-	}
 }
 
 func TestIntersectionCountArrayBitmap2(t *testing.T) {
@@ -1929,7 +1923,7 @@ func TestRunBinSearch(t *testing.T) {
 		},
 	}
 	for i, test := range tests {
-		idx, contains := runBinSearch(test.search, test.runs)
+		idx, contains := binSearchRuns(test.search, test.runs)
 		if !(test.exp == contains && test.expi == idx) {
 			t.Fatalf("test #%v expected (%v, %v) but got (%v, %v)", i, test.exp, test.expi, contains, idx)
 		}

From 88223627726eeb5687ec36417dd322e18874d431 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Tue, 20 Jun 2017 08:22:45 -0500
Subject: [PATCH 072/100] extending roaring test coverage

---
 roaring/roaring.go               | 24 +++++++-------
 roaring/roaring_internal_test.go | 29 +++++++++++++++++
 roaring/roaring_test.go          | 54 ++++++++++++++++++++++++++++++--
 3 files changed, 93 insertions(+), 14 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 80dbfa79a..49357620b 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -929,7 +929,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
 		itr.j++
 
 		// Find first non-zero bit in current bitmap, if possible.
-		hb := int(itr.j / 64)
+		hb := int(itr.j >> 6)
 
 		if hb >= len(c.bitmap) {
 			itr.i, itr.j = itr.i+1, -1
@@ -944,7 +944,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
 		// Otherwise iterate through remaining bitmaps to find next bit.
 		for hb++; hb < len(c.bitmap); hb++ {
 			if c.bitmap[hb] != 0 {
-				itr.j = int(hb*64) + trailingZeroN(c.bitmap[hb])
+				itr.j = int(hb<<6) + trailingZeroN(c.bitmap[hb])
 				return itr.peek(), false
 			}
 		}
@@ -1909,7 +1909,7 @@ func intersectionCountArrayBitmapOld(a, b *container) (n uint64) {
 
 func intersectionCountArrayBitmap(a, b *container) (n uint64) {
 	for _, val := range a.array {
-		i := val / 64
+		i := val >> 6
 		if i >= uint32(len(b.bitmap)) {
 			break
 		}
@@ -2052,8 +2052,8 @@ func intersectBitmapRun(a, b *container) *container {
 		}
 		for j := 0; j < len(b.runs); j++ {
 			vb := b.runs[j]
-			i := vb.start / 64 // index into a
-			vastart := 64 * i
+			i := vb.start >> 6 // index into a
+			vastart := i << 6
 			valast := vastart + 63
 			for valast >= vb.start && vastart <= vb.last {
 				if vastart >= vb.start && valast <= vb.last { // a within b
@@ -2077,7 +2077,7 @@ func intersectBitmapRun(a, b *container) *container {
 				}
 				// update loop vars
 				i++
-				vastart = 64 * i
+				vastart = i << 6
 				valast = vastart + 63
 			}
 		}
@@ -2300,8 +2300,8 @@ const maxBitmap = 0xFFFFFFFFFFFFFFFF
 
 // sets all bits in [i, j) (c must be a bitmap container)
 func (c *container) bitmapSetRange(i, j uint64) {
-	x := i / 64
-	y := (j - 1) / 64
+	x := i >> 6
+	y := (j - 1) >> 6
 	var X uint64 = maxBitmap << (i % 64)
 	var Y uint64 = maxBitmap >> (64 - (j % 64))
 	xcnt := popcnt(X)
@@ -2323,8 +2323,8 @@ func (c *container) bitmapSetRange(i, j uint64) {
 
 // xor's all bits in [i, j) with all true (c must be a bitmap container).
 func (c *container) bitmapXorRange(i, j uint64) {
-	x := i / 64
-	y := (j - 1) / 64
+	x := i >> 6
+	y := (j - 1) >> 6
 	var X uint64 = maxBitmap << (i % 64)
 	var Y uint64 = maxBitmap >> (64 - (j % 64))
 	if x == y {
@@ -2348,8 +2348,8 @@ func (c *container) bitmapXorRange(i, j uint64) {
 
 // zeroes all bits in [i, j) (c must be a bitmap container)
 func (c *container) bitmapZeroRange(i, j uint64) {
-	x := i / 64
-	y := (j - 1) / 64
+	x := i >> 6
+	y := (j - 1) >> 6
 	var X uint64 = maxBitmap << (i % 64)
 	var Y uint64 = maxBitmap >> (64 - (j % 64))
 	if x == y {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 291036ef8..0b103ae5f 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1929,3 +1929,32 @@ func TestRunBinSearch(t *testing.T) {
 		}
 	}
 }
+func TestBitmap_RemoveEmptyContainers(t *testing.T) {
+	bm1 := NewBitmap(1<<16, 2<<16, 3<<16)
+	bm1.Remove(2 << 16)
+	if bm1.countEmptyContainers() != 1 {
+		t.Fatalf("Should be 1 empty container ")
+	}
+	bm1.removeEmptyContainers()
+
+	if bm1.countEmptyContainers() != 0 {
+		t.Fatalf("Should be no empty containers ")
+	}
+}
+
+func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) {
+	bm1 := NewBitmap(1<<16, 2<<16, 3<<16)
+	bm1.Remove(2 << 16)
+	var buf bytes.Buffer
+	if _, err := bm1.WriteTo(&buf); err != nil {
+		t.Fatalf("Failure to write to bitmap buffer. ")
+	}
+	bm0 := NewBitmap()
+	bm0.UnmarshalBinary(buf.Bytes())
+	if bm0.countEmptyContainers() != 0 {
+		t.Fatalf("Should be no empty containers ")
+	}
+	if bm0.Count() != bm1.Count() {
+		t.Fatalf("Counts do not match after a marshal %d %d", bm0.Count(), bm1.Count())
+	}
+}
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 8150f4376..ba24ecd22 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -15,8 +15,8 @@
 package roaring_test
 
 import (
-	"fmt"
 	"bytes"
+	"fmt"
 	"math"
 	"math/rand"
 	"reflect"
@@ -166,7 +166,6 @@ func TestBitmap_CountRange(t *testing.T) {
 	if n := bm0.CountRange(0, 1); n != 1 {
 		t.Fatalf("unexpected n: %d", n)
 	}
-
 }
 
 func TestBitmap_Intersection(t *testing.T) {
@@ -418,6 +417,21 @@ func TestBitmap_IntersectionCount_BitmapBitmap(t *testing.T) {
 		t.Fatalf("unexpected n (reverse): %d", n)
 	}
 }
+func TestBitmap_IntersectionCount_Mixed(t *testing.T) {
+	bm0 := testBM()
+	bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536)
+	bm3 := roaring.NewBitmap(131072)
+
+	if n := bm0.IntersectionCount(bm0); n != bm0.Count() {
+		t.Fatalf("unexpected n: %d", n)
+	}
+	if n := bm0.IntersectionCount(bm1); n != 1 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+	if n := bm0.IntersectionCount(bm3); n != 1 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
 
 func TestBitmap_Quick_Array1(t *testing.T)     { testBitmapQuick(t, 1000, 1000, 2000) }
 func TestBitmap_Quick_Array2(t *testing.T)     { testBitmapQuick(t, 10000, 0, 1000) }
@@ -585,6 +599,42 @@ func TestIterator(t *testing.T) {
 	}
 }
 
+//testBM creates a bitmap with 3 containers(an array,bitmap, and run)
+func testBM() *roaring.Bitmap {
+	bm := roaring.NewBitmap()
+	//the array
+	for i := uint64(0); i < 1024; i += 4 {
+		bm.Add((1 << 16) + i)
+	}
+	//the bitmap
+	for i := uint64(0); i < 16384; i += 2 {
+		bm.Add((2 << 16) + i)
+	}
+	//small run
+	for i := uint64(0); i < 1024; i += 1 {
+		bm.Add((3 << 16) + i)
+	}
+	//large run
+	for i := uint64(0); i < 65535; i += 1 {
+		bm.Add((4 << 16) + i)
+	}
+	return bm
+}
+
+func TestBitmapOffsetRange(t *testing.T) {
+	bm := testBM()
+
+	bm1 := bm.OffsetRange(0, 0, 327680)
+	if bm1.Count() != bm.Count() {
+		t.Fatalf("Not Equal %d %d", bm1.Count(), bm.Count())
+	}
+	bm1 = bm.OffsetRange(0, 0, 131072)
+	if bm1.Count() != 256 {
+		t.Fatalf("Not Equal %d %d", bm1.Count(), 256)
+	}
+
+}
+
 var benchmarkBitmapIntersectionCountData struct {
 	a, b *roaring.Bitmap
 }

From 0b99941a104d1b4e6a98ec8e47164ae4ace0da40 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Tue, 20 Jun 2017 09:31:45 -0500
Subject: [PATCH 073/100] simplified interesect

---
 roaring/roaring.go | 43 +++++++++++++++++++++----------------------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 49357620b..2210da15f 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2090,22 +2090,23 @@ func intersectBitmapRun(a, b *container) *container {
 
 func intersectArrayBitmap(a, b *container) *container {
 	output := &container{}
-	itr := newBufIterator(newBitmapIterator(b.bitmap))
-	for i := 0; i < len(a.array); {
-		va := a.array[i]
-		vb, eof := itr.next()
-		if eof {
+	itra := newArrayIterator(a.array)
+	itrb := newBitmapIterator(b.bitmap)
+	va, eof1 := itra.next()
+	vb, eof2 := itrb.next()
+	for {
+		if eof1 || eof2 {
 			break
 		}
 
 		if va < vb {
-			i++
-			itr.unread()
+			va, eof1 = itra.next()
 		} else if va > vb {
-			// nop
+			vb, eof2 = itrb.next()
 		} else {
 			output.add(va)
-			i++
+			va, eof1 = itra.next()
+			vb, eof2 = itrb.next()
 		}
 	}
 	return output
@@ -2113,25 +2114,23 @@ func intersectArrayBitmap(a, b *container) *container {
 
 func intersectBitmapBitmap(a, b *container) *container {
 	output := &container{}
-	itr0 := newBufIterator(newBitmapIterator(a.bitmap))
-	itr1 := newBufIterator(newBitmapIterator(b.bitmap))
+	itr0 := newBitmapIterator(a.bitmap)
+	itr1 := newBitmapIterator(b.bitmap)
+	va, eof1 := itr0.next()
+	vb, eof2 := itr1.next()
 	for {
-		va, eof := itr0.next()
-		if eof {
-			break
-		}
-
-		vb, eof := itr1.next()
-		if eof {
+		if eof1 || eof2 {
 			break
 		}
 
 		if va < vb {
-			itr1.unread()
+			va, eof1 = itr0.next()
 		} else if va > vb {
-			itr0.unread()
+			vb, eof2 = itr1.next()
 		} else {
 			output.add(va)
+			va, eof1 = itr0.next()
+			vb, eof2 = itr1.next()
 		}
 	}
 	return output
@@ -3046,7 +3045,7 @@ func (itr *bitmapIterator) next() (v uint32, eof bool) {
 	itr.i++
 
 	// Find first non-zero bit in current bitmap, if possible.
-	hb := int(itr.i / 64)
+	hb := int(itr.i >> 6)
 	lb := itr.bitmap[hb] >> (uint(itr.i) % 64)
 	if lb != 0 {
 		itr.i = int(itr.i) + trailingZeroN(lb)
@@ -3056,7 +3055,7 @@ func (itr *bitmapIterator) next() (v uint32, eof bool) {
 	// Otherwise iterate through remaining bitmaps to find next bit.
 	for hb++; hb < len(itr.bitmap); hb++ {
 		if itr.bitmap[hb] != 0 {
-			itr.i = int(hb*64) + trailingZeroN(itr.bitmap[hb])
+			itr.i = int(hb<<6) + trailingZeroN(itr.bitmap[hb])
 			return uint32(itr.i), false
 		}
 	}

From eeaabff7fe78a4cf5c7f69d49771b3ebbc3f49be Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Tue, 20 Jun 2017 16:46:40 -0500
Subject: [PATCH 074/100] Standardize the default iterator value for j to -1
 for both arrays and runs

---
 roaring/roaring.go      | 18 ++++++++++++++----
 roaring/roaring_test.go | 12 +++++++++++-
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 80dbfa79a..bb802f00c 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -828,13 +828,13 @@ type BitmapInfo struct {
 // Iterator represents an iterator over a Bitmap.
 type Iterator struct {
 	bitmap  *Bitmap
-	i, j, k int // i: container; j: array index, bit index, or run index; k:
+	i, j, k int // i: container; j: array index, bit index, or run index; k: offset within the run
 }
 
 // eof returns true if the iterator is at the end of the bitmap.
 func (itr *Iterator) eof() bool { return itr.i >= len(itr.bitmap.containers) }
 
-// Seek moves to the first value equal to or greater than v.
+// Seek moves to the first value equal to or greater than `seek`.
 func (itr *Iterator) Seek(seek uint64) {
 	// Move to the correct container.
 	itr.i = search64(itr.bitmap.keys, highbits(seek))
@@ -845,7 +845,7 @@ func (itr *Iterator) Seek(seek uint64) {
 		return
 	}
 
-	// Move to the correct value index inside the array container.
+	// Move to the correct value index inside the container.
 	lb := lowbits(seek)
 	c := itr.bitmap.containers[itr.i]
 	if c.isArray() {
@@ -907,6 +907,16 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
 		}
 
 		if c.isRun() {
+			// Because itr.j for an array container defaults to -1
+			// but defaults to 0 for a run container, we need to
+			// standardize on treating -1 as our default value for itr.j.
+			// Note that this is easier than changing the default to 0
+			// because the array logic uses the negative number space
+			// to represent offsets to an array position that isn't filled
+			// (-1 being the first empty space in an array, or 0).
+			if itr.j == -1 {
+				itr.j++
+			}
 			r := c.runs[itr.j]
 			runLength := int(r.last - r.start)
 
@@ -917,7 +927,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
 
 			if itr.j >= len(c.runs) {
 				// Reached end of runs, move to the next container.
-				itr.i, itr.j = itr.i+1, 0
+				itr.i, itr.j = itr.i+1, -1
 				continue
 			}
 
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 8150f4376..98ffc8157 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -15,8 +15,8 @@
 package roaring_test
 
 import (
-	"fmt"
 	"bytes"
+	"fmt"
 	"math"
 	"math/rand"
 	"reflect"
@@ -78,6 +78,16 @@ func TestCheckRun(t *testing.T) {
 	}
 }
 
+// Ensure that we can transition between runs and arrays when materializing the bitmap.
+func TestContainerTransitions(t *testing.T) {
+	// [run, run][array][run]
+	b := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005)
+	b.Optimize() // convert to runs
+	if !reflect.DeepEqual(b.Slice(), []uint64{0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005}) {
+		t.Fatalf("unexpected slice: %+v", b.Slice())
+	}
+}
+
 // Ensure an empty bitmap returns false if checking for existence.
 func TestBitmap_Contains_Empty(t *testing.T) {
 	if roaring.NewBitmap().Contains(1000) {

From 0508a3c31f95d823f2e0e000c2e26a218e057e58 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Tue, 20 Jun 2017 18:19:58 -0500
Subject: [PATCH 075/100] fixed bug in difference;added more roaring test
 coverage

---
 roaring/roaring.go               |  12 +--
 roaring/roaring_internal_test.go | 134 +++++++++++++++++++++++++++++++
 roaring/roaring_test.go          |  28 +++++++
 3 files changed, 168 insertions(+), 6 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 2210da15f..a18a0ae09 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2479,7 +2479,7 @@ func differenceArrayRun(a, b *container) *container {
 	j := 0 // run index
 
 	// keep all array elements before beginning of runs
-	for ; i < int(b.runs[j].start); i++ {
+	for ; i < len(a.array) && a.array[i] < b.runs[j].start; i++ {
 		output.array = append(output.array, a.array[i])
 	}
 
@@ -2490,7 +2490,7 @@ func differenceArrayRun(a, b *container) *container {
 			output.array = append(output.array, a.array[i])
 		}
 		// update current run
-		if i >= int(b.runs[j].last) {
+		if a.array[i] >= b.runs[j].last {
 			j++
 			if j == len(b.runs) {
 				break
@@ -2498,10 +2498,10 @@ func differenceArrayRun(a, b *container) *container {
 		}
 	}
 	i++
-
-	// keep all array elements after end of runs
-	output.array = append(output.array, a.array[i:]...)
-
+	if i < len(a.array) {
+		// keep all array elements after end of runs
+		output.array = append(output.array, a.array[i:]...)
+	}
 	return output
 }
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 0b103ae5f..2a79a95b8 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -615,6 +615,140 @@ func TestIntersectBitmapRunArray(t *testing.T) {
 
 }
 
+func TestUnionMixed(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	c := &container{}
+
+	a.runs = []interval32{{start: 5, last: 10}}
+	a.n = 6
+	b.array = []uint32{1, 4, 5, 7, 10, 11, 12}
+	b.n = 7
+	res := union(a, b)
+	if !reflect.DeepEqual(res.array, []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, res.array)
+	}
+	res = union(b, a)
+	if !reflect.DeepEqual(res.array, []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}) {
+		t.Fatalf("test #2 expected %v, but got %v", []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, res.array)
+	}
+
+	res = union(a, a)
+	if !reflect.DeepEqual(res.runs, []interval32{{start: 5, last: 10}}) {
+		t.Fatalf("test #3 expected %v, but got %v", []interval32{{start: 5, last: 10}}, res.runs)
+	}
+	c.bitmap = []uint64{0x3}
+	c.n = 2
+	res = union(c, a)
+	if !reflect.DeepEqual(res.bitmap, []uint64{2019}) {
+		t.Fatalf("test #4 expected %v, but got %v", []uint64{2019}, res.bitmap)
+	}
+	res = union(a, c)
+	if !reflect.DeepEqual(res.bitmap, []uint64{2019}) {
+		t.Fatalf("test #5 expected %v, but got %v", []uint64{2019}, res.bitmap)
+	}
+	res = union(b, c)
+	if !reflect.DeepEqual(res.array, []uint32{0, 1, 4, 5, 7, 10, 11, 12}) {
+		t.Fatalf("test #6 expected %v, but got %v", []uint32{0, 1, 4, 5, 7, 10, 11, 12}, res.array)
+	}
+	res = union(c, b)
+	if !reflect.DeepEqual(res.array, []uint32{0, 1, 4, 5, 7, 10, 11, 12}) {
+		t.Fatalf("test #6 expected %v, but got %v", []uint32{0, 1, 4, 5, 7, 10, 11, 12}, res.array)
+	}
+
+}
+func TestIntersectMixed(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	c := &container{}
+
+	a.runs = []interval32{{start: 5, last: 10}}
+	a.n = 6
+	b.array = []uint32{1, 4, 5, 7, 10, 11, 12}
+	b.n = 7
+	res := intersect(a, b)
+	if !reflect.DeepEqual(res.array, []uint32{5, 7, 10}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint32{5, 7, 10}, res.array)
+	}
+	res = intersect(b, a)
+	if !reflect.DeepEqual(res.array, []uint32{5, 7, 10}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint32{5, 7, 10}, res.array)
+	}
+
+	res = intersect(a, a)
+	if !reflect.DeepEqual(res.runs, []interval32{{start: 5, last: 10}}) {
+		t.Fatalf("test #3 expected %v, but got %v", []interval32{{start: 5, last: 10}}, res.runs)
+	}
+	c.bitmap = []uint64{0x60}
+	c.n = 2
+	res = intersect(c, a)
+	if !reflect.DeepEqual(res.array, []uint32{5, 6}) {
+		t.Fatalf("test #4 expected %v, but got %v", []uint32{6}, res.array)
+	}
+
+	res = intersect(a, c)
+	if !reflect.DeepEqual(res.array, []uint32{5, 6}) {
+		t.Fatalf("test #5 expected %v, but got %v", []uint32{6}, res.array)
+	}
+
+	res = intersect(b, c)
+	if !reflect.DeepEqual(res.array, []uint32{5}) {
+		t.Fatalf("test #6 expected %v, but got %v", []uint32{5}, res.array)
+	}
+	res = intersect(c, b)
+	if !reflect.DeepEqual(res.array, []uint32{5}) {
+		t.Fatalf("test #7 expected %v, but got %v", []uint32{5}, res.array)
+	}
+
+}
+func TestDifferenceMixed(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	c := &container{}
+
+	a.runs = []interval32{{start: 5, last: 10}}
+	a.n = a.runCountRange(0, 100)
+
+	b.array = []uint32{1, 4, 5, 7, 10, 11, 12}
+	b.n = len(b.array)
+	res := difference(a, b)
+
+	if !reflect.DeepEqual(res.array, []uint32{6, 8, 9}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint32{6, 8, 9}, res.runs)
+	}
+	res = difference(b, a)
+	if !reflect.DeepEqual(res.array, []uint32{1, 4, 11, 12}) {
+		t.Fatalf("test #2 expected %v, but got %v", []uint32{1, 4, 11, 12}, res.array)
+	}
+
+	res = difference(a, a)
+	if !reflect.DeepEqual(res.runs, []interval32{}) {
+		t.Fatalf("test #3 expected empty but got %v", res.runs)
+	}
+	c.bitmap = []uint64{0x64}
+	c.n = c.countRange(0, 100)
+	res = difference(c, a)
+
+	if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) {
+		t.Fatalf("test #4 expected %v, but got %v", []uint32{4}, res.bitmap)
+	}
+
+	res = difference(a, c)
+	if !reflect.DeepEqual(res.runs, []interval32{{start: 7, last: 10}}) {
+		t.Fatalf("test #5 expected %v, but got %v", []interval32{{start: 7, last: 10}}, res.runs)
+	}
+
+	res = difference(b, c)
+	if !reflect.DeepEqual(res.array, []uint32{1, 4, 7, 10, 11, 12}) {
+		t.Fatalf("test #6 expected %v, but got %v", []uint32{1, 4, 7, 10, 11, 12}, res.array)
+	}
+	res = difference(c, b)
+	if !reflect.DeepEqual(res.array, []uint32{2, 6}) {
+		t.Fatalf("test #7 expected %v, but got %v", []uint32{2, 6}, res.array)
+	}
+
+}
+
 func TestUnionRunRun(t *testing.T) {
 	a := &container{}
 	b := &container{}
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index ba24ecd22..a86f636cb 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -194,6 +194,11 @@ func TestBitmap_Union1(t *testing.T) {
 	if n := result.Count(); n != 2682675 {
 		t.Fatalf("unexpected n: %d", n)
 	}
+	bm := testBM()
+	result = bm.Union(bm0)
+	if n := result.Count(); n != 75009 {
+		t.Fatalf("unexpected n: %d", n)
+	}
 
 }
 
@@ -237,6 +242,15 @@ func TestBitmap_Union(t *testing.T) {
 	}
 }
 
+func TestBitmap_Xor(t *testing.T) {
+	bm0 := testBM()
+	bm1 := roaring.NewBitmap(0, 1, 2, 3)
+	result := bm1.Xor(bm0)
+	if n := result.Count(); n != 75011 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
+
 func TestBitmap_Xor_ArrayArray(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003)
 	bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002)
@@ -601,6 +615,7 @@ func TestIterator(t *testing.T) {
 
 //testBM creates a bitmap with 3 containers(an array,bitmap, and run)
 func testBM() *roaring.Bitmap {
+
 	bm := roaring.NewBitmap()
 	//the array
 	for i := uint64(0); i < 1024; i += 4 {
@@ -618,6 +633,7 @@ func testBM() *roaring.Bitmap {
 	for i := uint64(0); i < 65535; i += 1 {
 		bm.Add((4 << 16) + i)
 	}
+	//count 75007
 	return bm
 }
 
@@ -635,6 +651,10 @@ func TestBitmapOffsetRange(t *testing.T) {
 
 }
 
+func TestBitmapBufIterator(t *testing.T) {
+
+}
+
 var benchmarkBitmapIntersectionCountData struct {
 	a, b *roaring.Bitmap
 }
@@ -706,3 +726,11 @@ func diff(a, b []uint64) string {
 	}
 	return ""
 }
+
+func TestBitmap_Intersect(t *testing.T) {
+	bm0 := testBM()
+	result := bm0.Intersect(bm0)
+	if bm0.Count() != result.Count() {
+		t.Fatalf("Counts do not match %d %d", bm0.Count(), result.Count())
+	}
+}

From a387deefb17c4567289e7184adec4995aa694390 Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Wed, 21 Jun 2017 09:20:54 -0500
Subject: [PATCH 076/100] Test the case where last and first bits of adjoining
 containers are set.

---
 roaring/roaring_test.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 98ffc8157..bb9f5a21c 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -86,6 +86,14 @@ func TestContainerTransitions(t *testing.T) {
 	if !reflect.DeepEqual(b.Slice(), []uint64{0, 1, 2, 3, 4, 5, 1000, 1001, 1002, 1003, 1004, 1005, 100000, 100001, 100002, 132000, 132001, 132002, 132003, 132004, 132005}) {
 		t.Fatalf("unexpected slice: %+v", b.Slice())
 	}
+
+	// Test the case where last and first bits of adjoining containers are set.
+	// [run][array][run]
+	b2 := roaring.NewBitmap(65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076)
+	b2.Optimize() // convert to runs
+	if !reflect.DeepEqual(b2.Slice(), []uint64{65531, 65532, 65533, 65534, 65535, 65536, 131071, 131072, 131073, 131074, 131075, 131076}) {
+		t.Fatalf("unexpected slice: %+v", b2.Slice())
+	}
 }
 
 // Ensure an empty bitmap returns false if checking for existence.

From 4da0c167e1ca11e5138828bf8aba0898c66aacb2 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Wed, 21 Jun 2017 20:18:03 -0500
Subject: [PATCH 077/100] extended tests for difference operation

---
 roaring/roaring_internal_test.go | 47 +++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 10 deletions(-)

diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 2a79a95b8..48b9268e0 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -705,30 +705,36 @@ func TestDifferenceMixed(t *testing.T) {
 	a := &container{}
 	b := &container{}
 	c := &container{}
+	d := &container{}
 
 	a.runs = []interval32{{start: 5, last: 10}}
 	a.n = a.runCountRange(0, 100)
 
-	b.array = []uint32{1, 4, 5, 7, 10, 11, 12}
+	b.array = []uint32{0, 2, 4, 6, 8, 10, 12}
 	b.n = len(b.array)
+
+	d.array = []uint32{1, 3, 5, 7, 9, 11, 12}
+	d.n = len(d.array)
+
 	res := difference(a, b)
 
-	if !reflect.DeepEqual(res.array, []uint32{6, 8, 9}) {
-		t.Fatalf("test #1 expected %v, but got %v", []uint32{6, 8, 9}, res.runs)
+	if !reflect.DeepEqual(res.array, []uint32{5, 7, 9}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint32{5, 7, 9}, res.array)
 	}
+
 	res = difference(b, a)
-	if !reflect.DeepEqual(res.array, []uint32{1, 4, 11, 12}) {
-		t.Fatalf("test #2 expected %v, but got %v", []uint32{1, 4, 11, 12}, res.array)
+	if !reflect.DeepEqual(res.array, []uint32{0, 2, 4, 12}) {
+		t.Fatalf("test #2 expected %v, but got %v", []uint32{0, 2, 4, 12}, res.array)
 	}
 
 	res = difference(a, a)
 	if !reflect.DeepEqual(res.runs, []interval32{}) {
 		t.Fatalf("test #3 expected empty but got %v", res.runs)
 	}
+
 	c.bitmap = []uint64{0x64}
 	c.n = c.countRange(0, 100)
 	res = difference(c, a)
-
 	if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) {
 		t.Fatalf("test #4 expected %v, but got %v", []uint32{4}, res.bitmap)
 	}
@@ -739,12 +745,33 @@ func TestDifferenceMixed(t *testing.T) {
 	}
 
 	res = difference(b, c)
-	if !reflect.DeepEqual(res.array, []uint32{1, 4, 7, 10, 11, 12}) {
-		t.Fatalf("test #6 expected %v, but got %v", []uint32{1, 4, 7, 10, 11, 12}, res.array)
+	if !reflect.DeepEqual(res.array, []uint32{0, 4, 8, 10, 12}) {
+		t.Fatalf("test #6 expected %v, but got %v", []uint32{0, 4, 8, 10, 12}, res.array)
 	}
+
 	res = difference(c, b)
-	if !reflect.DeepEqual(res.array, []uint32{2, 6}) {
-		t.Fatalf("test #7 expected %v, but got %v", []uint32{2, 6}, res.array)
+	if !reflect.DeepEqual(res.array, []uint32{5}) {
+		t.Fatalf("test #7 expected %v, but got %v", []uint32{5}, res.array)
+	}
+
+	res = difference(b, b)
+	if res.n != 0 {
+		t.Fatalf("test #8 expected 0, but got %d", res.n)
+	}
+
+	res = difference(c, c)
+	if res.n != 0 {
+		t.Fatalf("test #9 expected 0, but got %d", res.n)
+	}
+
+	res = difference(d, b)
+	if !reflect.DeepEqual(res.array, []uint32{1, 3, 5, 7, 9, 11}) {
+		t.Fatalf("test #10 expected %v, but got %d", []uint32{1, 3, 5, 7, 9, 11}, res.array)
+	}
+
+	res = difference(b, d)
+	if !reflect.DeepEqual(res.array, []uint32{0, 2, 4, 6, 8, 10}) {
+		t.Fatalf("test #11 expected %v, but got %d", []uint32{0, 2, 4, 6, 8, 10}, res.array)
 	}
 
 }

From 188a4aa8d9a91f9e302244a178e613b87e64ac6a Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Thu, 22 Jun 2017 10:58:47 -0500
Subject: [PATCH 078/100] extended bitmap contains test coverage

---
 roaring/roaring_test.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index bc706cf84..bafe3daad 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -651,6 +651,7 @@ func testBM() *roaring.Bitmap {
 	for i := uint64(0); i < 65535; i += 1 {
 		bm.Add((4 << 16) + i)
 	}
+	bm.Optimize()
 	//count 75007
 	return bm
 }
@@ -668,6 +669,19 @@ func TestBitmapOffsetRange(t *testing.T) {
 	}
 
 }
+func TestBitmapContains(t *testing.T) {
+	bm := testBM()
+
+	//search for run value present
+	if found := bm.Contains(3 << 16); !found {
+		t.Fatalf("Test #1 Not Found %d ", 3<<16)
+	}
+
+	//search for value not present
+	if found := bm.Contains((3 << 16) + 2048); found {
+		t.Fatalf("Test #2 Found %d ", (3<<16)+2048)
+	}
+}
 
 func TestBitmapBufIterator(t *testing.T) {
 

From 10490a3402b50df35c61b358c5325f4f63bf2966 Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Thu, 22 Jun 2017 11:03:05 -0500
Subject: [PATCH 079/100] Improves test coverage for `IntersectionCount`
 operations

---
 roaring/roaring_test.go | 45 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index bc706cf84..8b784ff0f 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -403,7 +403,7 @@ func TestBitmap_Flip_After(t *testing.T) {
 
 // Ensure bitmap can return the number of intersecting bits in two bitmaps.
 func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) {
-	bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003)
+	bm0 := roaring.NewBitmap(0, 1, 1000001, 1000002, 1000003)
 	bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002)
 
 	if n := bm0.IntersectionCount(bm1); n != 3 {
@@ -413,6 +413,49 @@ func TestBitmap_IntersectionCount_ArrayArray(t *testing.T) {
 	}
 }
 
+// Ensure bitmap can return the number of intersecting bits in two bitmaps.
+func TestBitmap_IntersectionCount_ArrayRun(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003)
+	bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006)
+	bm1.Optimize() // convert to runs
+
+	if n := bm0.IntersectionCount(bm1); n != 3 {
+		t.Fatalf("unexpected n: %d", n)
+	} else if n := bm1.IntersectionCount(bm0); n != 3 {
+		t.Fatalf("unexpected n (reverse): %d", n)
+	}
+}
+
+// Ensure bitmap can return the number of intersecting bits in two bitmaps.
+func TestBitmap_IntersectionCount_RunRun(t *testing.T) {
+	bm0 := roaring.NewBitmap(3, 4, 5, 6, 7, 8, 1000001, 1000002, 1000003, 1000004)
+	bm0.Optimize() // convert to runs
+	bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006)
+	bm1.Optimize() // convert to runs
+
+	if n := bm0.IntersectionCount(bm1); n != 6 {
+		t.Fatalf("unexpected n: %d", n)
+	} else if n := bm1.IntersectionCount(bm0); n != 6 {
+		t.Fatalf("unexpected n (reverse): %d", n)
+	}
+}
+
+// Ensure bitmap can return the number of intersecting bits in two bitmaps.
+func TestBitmap_IntersectionCount_BitmapRun(t *testing.T) {
+	bm0 := roaring.NewBitmap()
+	for i := uint64(3); i <= 1000006; i += 2 {
+		bm0.Add(i)
+	}
+	bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006)
+	bm1.Optimize() // convert to runs
+
+	if n := bm0.IntersectionCount(bm1); n != 4 {
+		t.Fatalf("unexpected n: %d", n)
+	} else if n := bm1.IntersectionCount(bm0); n != 4 {
+		t.Fatalf("unexpected n (reverse): %d", n)
+	}
+}
+
 // Ensure bitmap can return the number of intersecting bits in two bitmaps.
 func TestBitmap_IntersectionCount_ArrayBitmap(t *testing.T) {
 	bm0 := roaring.NewBitmap(1, 70, 200, 4097, 4098)

From 087c78a0dfb98e2598cb48c54331a53ba31576b0 Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Thu, 22 Jun 2017 12:04:01 -0500
Subject: [PATCH 080/100] add test coverage for `CountRange` operations

---
 roaring/roaring_test.go | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index a097e1fbe..c9b18c9a2 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -163,7 +163,8 @@ func TestBitmap_Max(t *testing.T) {
 		}
 	}
 }
-func TestBitmap_CountRange(t *testing.T) {
+
+func TestBitmap_BitmapCountRange(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 2683177)
 	for i := uint64(628); i < 2683301; i++ {
 		bm0.Add(i)
@@ -184,6 +185,32 @@ func TestBitmap_CountRange(t *testing.T) {
 	if n := bm0.CountRange(0, 1); n != 1 {
 		t.Fatalf("unexpected n: %d", n)
 	}
+
+	// Test the case where the range is outside of the bitmap space.
+	if n := bm0.CountRange(10000000, 10000001); n != 0 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
+
+func TestBitmap_ArrayCountRange(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 2683177, 2683313)
+	if n := bm0.CountRange(1, 2683311); n != 1 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
+
+func TestBitmap_RunCountRange(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17, 1000000, 1000002, 1000003, 1000004, 1000005, 1000006, 1000010, 1000011, 1000012, 1000013, 1000014)
+	bm0.Optimize() // convert to runs
+	if n := bm0.CountRange(15, 1000003); n != 5 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
+	bm1 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)
+	bm1.Optimize() // convert to runs
+	if n := bm1.CountRange(5, 12); n != 7 {
+		t.Fatalf("unexpected n: %d", n)
+	}
 }
 
 func TestBitmap_Intersection(t *testing.T) {

From 1e0a529b4efd84a7dee4ae3aa39275a4f265a464 Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Thu, 22 Jun 2017 13:59:19 -0500
Subject: [PATCH 081/100] improve coverage in intersect** operations

---
 roaring/roaring_test.go | 66 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index c9b18c9a2..c8a19002b 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -258,6 +258,70 @@ func TestBitmap_Intersection_Empty(t *testing.T) {
 
 }
 
+func TestBitmap_IntersectArrayArray(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 1, 2683, 5005)
+	bm1 := roaring.NewBitmap(0, 2683, 2684, 5000)
+
+	result := bm0.Intersect(bm1)
+	if n := result.Count(); n != 2 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
+
+func TestBitmap_IntersectBitmapBitmap(t *testing.T) {
+	bm0 := roaring.NewBitmap()
+	for i := uint64(0); i < 65536; i += 2 {
+		bm0.Add(i)
+	}
+
+	bm1 := roaring.NewBitmap()
+	for i := uint64(0); i < 65536; i += 3 {
+		bm1.Add(i)
+	}
+
+	result := bm0.Intersect(bm1)
+	if n := result.Count(); n != 10923 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
+
+func TestBitmap_IntersectRunRun(t *testing.T) {
+	// Intersect two runs that result in an array.
+	bm0 := roaring.NewBitmap(0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15)
+	bm0.Optimize() // convert to runs
+	bm1 := roaring.NewBitmap(5, 6, 7, 8, 9, 10, 11)
+	bm1.Optimize() // convert to runs
+	result := bm0.Intersect(bm1)
+	if n := result.Count(); n != 3 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
+	// Intersect two runs that result in a bitmap.
+	bm2 := roaring.NewBitmap()
+	runLen := uint64(25)
+	spaceLen := uint64(8)
+	offset := (runLen / 2) + spaceLen
+	for i := uint64(0); i < (65536 - runLen - offset); i += (runLen + spaceLen) {
+		for j := uint64(0); j < runLen; j++ {
+			bm2.Add(offset + i + j)
+		}
+	}
+	bm2.Optimize() // convert to runs
+	bm3 := roaring.NewBitmap()
+	runLen = uint64(32)
+	spaceLen = uint64(1)
+	for i := uint64(0); i < (65536 - runLen); i += (runLen + spaceLen) {
+		for j := uint64(0); j < runLen; j++ {
+			bm3.Add(i + j)
+		}
+	}
+	bm3.Optimize() // convert to runs
+	result = bm2.Intersect(bm3)
+	if n := result.Count(); n != 47628 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
+
 func TestBitmap_Difference(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 2683177)
 	bm1 := roaring.NewBitmap()
@@ -701,7 +765,7 @@ func TestIterator(t *testing.T) {
 	}
 }
 
-//testBM creates a bitmap with 3 containers(an array,bitmap, and run)
+// testBM creates a bitmap with 3 containers: array, bitmap, and run.
 func testBM() *roaring.Bitmap {
 
 	bm := roaring.NewBitmap()

From 8fd752ddb14461896d03c210142742060ab68b20 Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Thu, 22 Jun 2017 16:01:52 -0500
Subject: [PATCH 082/100] bug fix in `differenceArrayRun` logic. add more test
 coverage for `differenceArrayRun`

---
 roaring/roaring.go      | 33 ++++++++++++++++++++-------------
 roaring/roaring_test.go | 28 ++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 13 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 66ce116eb..653831940 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -453,7 +453,6 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
 		}
 
 	}
-
 	return output
 }
 
@@ -2488,29 +2487,37 @@ func differenceArrayRun(a, b *container) *container {
 	i := 0 // array index
 	j := 0 // run index
 
-	// keep all array elements before beginning of runs
-	for ; i < len(a.array) && a.array[i] < b.runs[j].start; i++ {
-		output.array = append(output.array, a.array[i])
-	}
-
 	// handle overlap
-	for ; i < a.n; i++ {
-		// if array element in run, keep
-		if !(a.array[i] >= b.runs[j].start && a.array[i] <= b.runs[j].last) {
-			output.array = append(output.array, a.array[i])
+	for i < a.n {
+
+		// keep all array elements before beginning of runs
+		if a.array[i] < b.runs[j].start {
+			output.add(a.array[i])
+			i++
+			continue
 		}
-		// update current run
-		if a.array[i] >= b.runs[j].last {
+
+		// if array element in run, skip it
+		if a.array[i] >= b.runs[j].start && a.array[i] <= b.runs[j].last {
+			i++
+			continue
+		}
+
+		// if array element larger than current run, check next run
+		if a.array[i] > b.runs[j].last {
 			j++
 			if j == len(b.runs) {
 				break
 			}
 		}
 	}
-	i++
+
 	if i < len(a.array) {
 		// keep all array elements after end of runs
 		output.array = append(output.array, a.array[i:]...)
+		// TODO: consider handling container.n mutations in one place
+		// like we do with container.add().
+		output.n += len(a.array[i:])
 	}
 	return output
 }
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index a097e1fbe..fc0a48966 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -242,6 +242,7 @@ func TestBitmap_Difference(t *testing.T) {
 		t.Fatalf("unexpected n: %d", n)
 	}
 }
+
 func TestBitmap_Difference_Empty(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 2683177)
 	bm1 := roaring.NewBitmap()
@@ -251,6 +252,33 @@ func TestBitmap_Difference_Empty(t *testing.T) {
 	}
 }
 
+func TestBitmap_DifferenceArrayArray(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20)
+	bm1 := roaring.NewBitmap(1, 3, 6, 9, 12, 15, 18)
+	result := bm0.Difference(bm1)
+	if n := result.Count(); n != 5 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
+
+func TestBitmap_DifferenceArrayRun(t *testing.T) {
+	bm0 := roaring.NewBitmap(0, 4, 8, 12, 16, 20, 36, 40, 44)
+
+	bm1 := roaring.NewBitmap(1, 2, 3, 4, 5, 6, 7, 8, 9, 30, 31, 32, 33, 34, 35, 36)
+	bm1.Optimize() // convert to runs
+	result := bm0.Difference(bm1)
+	if n := result.Count(); n != 6 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
+	// ensure empty array returns empty
+	bm2 := roaring.NewBitmap()
+	result = bm2.Difference(bm1)
+	if n := result.Count(); n != 0 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+}
+
 func TestBitmap_Union(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 1000001, 1000002, 1000003)
 	bm1 := roaring.NewBitmap(0, 50000, 1000001, 1000002)

From fc6898a9337b40432be1da1b23c14d4fd1037dae Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Thu, 22 Jun 2017 16:07:21 -0500
Subject: [PATCH 083/100] remove a test that was not testing the correct thing

---
 roaring/roaring_test.go | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index fc0a48966..98c789043 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -270,13 +270,6 @@ func TestBitmap_DifferenceArrayRun(t *testing.T) {
 	if n := result.Count(); n != 6 {
 		t.Fatalf("unexpected n: %d", n)
 	}
-
-	// ensure empty array returns empty
-	bm2 := roaring.NewBitmap()
-	result = bm2.Difference(bm1)
-	if n := result.Count(); n != 0 {
-		t.Fatalf("unexpected n: %d", n)
-	}
 }
 
 func TestBitmap_Union(t *testing.T) {

From b14a31dab3628e00c9bc4240d8d1553f418654ac Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Thu, 22 Jun 2017 16:21:19 -0500
Subject: [PATCH 084/100] fixed bug with xor; extended test coverage for xor

---
 roaring/roaring.go               | 15 ++++++++--
 roaring/roaring_internal_test.go | 48 +++++++++++++++++++++++++++++---
 roaring/roaring_test.go          | 23 +++++++++++++--
 3 files changed, 78 insertions(+), 8 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 66ce116eb..ccbc3fc3c 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2751,12 +2751,24 @@ func xor(a, b *container) *container {
 	if a.isArray() {
 		if b.isArray() {
 			return xorArrayArray(a, b)
+		} else if b.isRun() {
+			return xorArrayRun(a, b)
 		} else {
 			return xorArrayBitmap(a, b)
 		}
+	} else if a.isRun() {
+		if b.isArray() {
+			return xorArrayRun(b, a)
+		} else if b.isRun() {
+			return xorRunRun(a, b)
+		} else {
+			return xorBitmapRun(b, a)
+		}
 	} else {
 		if b.isArray() {
 			return xorArrayBitmap(b, a)
+		} else if b.isRun() {
+			return xorBitmapRun(a, b)
 		} else {
 			return xorBitmapBitmap(a, b)
 		}
@@ -2813,7 +2825,6 @@ func xorBitmapBitmap(a, b *container) *container {
 	output := &container{
 		bitmap: make([]uint64, bitmapN),
 	}
-
 	for i := 0; i < bitmapN; i++ {
 		v := a.bitmap[i] ^ b.bitmap[i]
 		output.bitmap[i] = v
@@ -3232,7 +3243,7 @@ func xorCompare(x *xorstm) (r1 interval32, has_data bool) {
 		has_data = true
 	} else if x.vb.last < x.va.start { //vb before
 		x.vb_valid = false
-		r1 = x.va
+		r1 = x.vb
 		has_data = true
 	} else if x.va.start == x.vb.start && x.va.last == x.vb.last { // Equal
 		x.va_valid = false
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 48b9268e0..3cb9742c5 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1624,9 +1624,41 @@ func TestXorArrayRun(t *testing.T) {
 	b := &container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
 	exp := []uint32{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}
 
-	ret := xorArrayRun(a, b)
+	//ret := xorArrayRun(a, b)
+	ret := xor(a, b)
 	if !reflect.DeepEqual(ret.array, exp) {
-		t.Fatalf("test expected %v, but got %v", exp, ret.array)
+		t.Fatalf("test #1 expected %v, but got %v", exp, ret.array)
+	}
+
+	ret = xor(b, a)
+	if !reflect.DeepEqual(ret.array, exp) {
+		t.Fatalf("test #2 expected %v, but got %v", exp, ret.array)
+	}
+	c := &container{array: []uint32{1, 5, 10, 11, 12, 13, 14}}
+	exp = []uint32{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}
+	ret = xor(b, c)
+	if !reflect.DeepEqual(ret.array, exp) {
+		t.Fatalf("test #3 expected %v, but got %v", exp, ret.array)
+	}
+	ret = xor(c, b)
+	if !reflect.DeepEqual(ret.array, exp) {
+		t.Fatalf("test #4 expected %v, but got %v", exp, ret.array)
+	}
+}
+
+//special case that didn't fit the xorrunrun table testing below.
+func TestXorRunRun1(t *testing.T) {
+	a := &container{}
+	b := &container{}
+	a.runs = []interval32{{start: 4, last: 10}}
+	b.runs = []interval32{{start: 5, last: 10}}
+	ret := xorRunRun(a, b)
+	if !reflect.DeepEqual(ret.array, []uint32{4}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint32{4}, ret.array)
+	}
+	ret = xorRunRun(b, a)
+	if !reflect.DeepEqual(ret.array, []uint32{4}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint32{4}, ret.array)
 	}
 }
 
@@ -1716,6 +1748,10 @@ func TestXorRunRun(t *testing.T) {
 		if !reflect.DeepEqual(ret.runs, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs)
 		}
+		ret = xorRunRun(b, a)
+		if !reflect.DeepEqual(ret.runs, test.exp) {
+			t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret.runs)
+		}
 	}
 }
 
@@ -1804,11 +1840,15 @@ func TestXorBitmapRun(t *testing.T) {
 	for i, test := range tests {
 		a.bitmap = test.bitmap
 		b.runs = test.runs
-		ret := xorBitmapRun(a, b)
-
+		//xorBitmapRun
+		ret := xor(a, b)
 		if !reflect.DeepEqual(ret.bitmap, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap)
 		}
+		ret = xor(b, a)
+		if !reflect.DeepEqual(ret.bitmap, test.exp) {
+			t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret.bitmap)
+		}
 	}
 
 }
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index bafe3daad..7414651dd 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -217,6 +217,10 @@ func TestBitmap_Union1(t *testing.T) {
 	if n := result.Count(); n != 75009 {
 		t.Fatalf("unexpected n: %d", n)
 	}
+	result = bm.Union(bm)
+	if n := result.Count(); n != 75007 {
+		t.Fatalf("unexpected n: %d", n)
+	}
 
 }
 
@@ -267,6 +271,16 @@ func TestBitmap_Xor(t *testing.T) {
 	if n := result.Count(); n != 75011 {
 		t.Fatalf("unexpected n: %d", n)
 	}
+
+	result = bm0.Xor(bm1)
+	if n := result.Count(); n != 75011 {
+		t.Fatalf("unexpected n: %d", n)
+	}
+
+	result = bm0.Xor(bm0)
+	if n := result.Count(); n != 0 {
+		t.Fatalf("unexpected n: %d", n)
+	}
 }
 
 func TestBitmap_Xor_ArrayArray(t *testing.T) {
@@ -304,13 +318,18 @@ func TestBitmap_Xor_ArrayBitmap(t *testing.T) {
 
 	result := bm0.Xor(bm1)
 	if n := result.Count(); n != 4999 {
-		t.Fatalf("unexpected n: %d", n)
+		t.Fatalf("test #1 unexpected n: %d", n)
+	}
+
+	result = bm1.Xor(bm0)
+	if n := result.Count(); n != 4999 {
+		t.Fatalf("test #2 unexpected n: %d", n)
 	}
 
 	//equivalence bitmap test
 	result = result.Xor(result)
 	if n := result.Count(); n > 0 {
-		t.Fatalf("unexpected n: %d", n)
+		t.Fatalf("test 3 unexpected n: %d", n)
 	}
 
 	empty := roaring.NewBitmap()

From bed7b41faf0f01d071ecda7261fea0cb5b491792 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 23 Jun 2017 08:13:57 -0500
Subject: [PATCH 085/100] fixed memory hit on startup

---
 fragment.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fragment.go b/fragment.go
index 4758f18e8..2b923507b 100644
--- a/fragment.go
+++ b/fragment.go
@@ -272,8 +272,8 @@ func (f *Fragment) openCache() error {
 	// Read in all rows by ID.
 	// This will cause them to be added to the cache.
 	for _, id := range pb.IDs {
-		//n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth)
-		n := f.row(id, true, true).Count()
+		n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth)
+		//n := f.row(id, true, true).Count()
 		f.cache.BulkAdd(id, n)
 	}
 	f.cache.Invalidate()

From c8ef6fde89d1ebdea16c42b5dfbb71b98422592c Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Fri, 23 Jun 2017 10:24:43 -0500
Subject: [PATCH 086/100] test edge case (end exclusive) in `arrayCountRange`
 for range [start, end)

---
 roaring/roaring_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index c8a19002b..7c2c2edf2 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -194,7 +194,7 @@ func TestBitmap_BitmapCountRange(t *testing.T) {
 
 func TestBitmap_ArrayCountRange(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 2683177, 2683313)
-	if n := bm0.CountRange(1, 2683311); n != 1 {
+	if n := bm0.CountRange(1, 2683313); n != 1 {
 		t.Fatalf("unexpected n: %d", n)
 	}
 }

From 43c275c6e2152a6609e26c046891309e5f79c572 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 23 Jun 2017 16:03:56 -0500
Subject: [PATCH 087/100] extended bitmap.optimize() to look at run containers

---
 roaring/roaring.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 87a8bf277..03a9430e2 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1328,10 +1328,15 @@ func (c *container) Optimize() {
 		}
 	} else if c.isBitmap() {
 		runs := c.bitmapCountRuns()
-		if runs < 2048 {
+		if runs < RunMaxSize {
 			c.bitmapToRun()
 		}
+	} else if c.isRun() {
+		if len(c.runs) > RunMaxSize {
+			c.runToBitmap()
+		}
 	}
+
 }
 
 func (c *container) arrayContains(v uint32) bool {

From c3a648b3f85004775f3b944b3c21322a256a4e87 Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Fri, 23 Jun 2017 16:55:49 -0500
Subject: [PATCH 088/100] remove unused roaring.BufIterator. test the unread()
 panic case in bufBitmapIterator

---
 roaring/roaring.go               | 66 ++++----------------------------
 roaring/roaring_internal_test.go | 59 ++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 58 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 87a8bf277..7a294d118 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -976,56 +976,6 @@ func (itr *Iterator) peek() uint64 {
 	return uint64(key)<<16 | uint64(itr.j)
 }
 
-// BufIterator wraps an iterator to provide the ability to unread values.
-type BufIterator struct {
-	buf struct {
-		v    uint64
-		eof  bool
-		full bool
-	}
-	itr *Iterator
-}
-
-// NewBufIterator returns a buffered iterator that wraps itr.
-func NewBufIterator(itr *Iterator) *BufIterator {
-	return &BufIterator{itr: itr}
-}
-
-// Seek moves to the first pair equal to or greater than pseek/bseek.
-func (itr *BufIterator) Seek(v uint64) {
-	itr.buf.full = false
-	itr.itr.Seek(v)
-}
-
-// Next returns the next pair in the bitmap.
-// If a value has been buffered then it is returned and the buffer is cleared.
-func (itr *BufIterator) Next() (v uint64, eof bool) {
-	if itr.buf.full {
-		itr.buf.full = false
-		return itr.buf.v, itr.buf.eof
-	}
-
-	// Read value onto buffer in case of unread.
-	itr.buf.v, itr.buf.eof = itr.itr.Next()
-	return itr.buf.v, itr.buf.eof
-}
-
-// Peek reads the next value but leaves it on the buffer.
-func (itr *BufIterator) Peek() (v uint64, eof bool) {
-	v, eof = itr.Next()
-	itr.Unread()
-	return
-}
-
-// Unread pushes previous pair on to the buffer.
-// Panics if the buffer is already full.
-func (itr *BufIterator) Unread() {
-	if itr.buf.full {
-		panic("roaring.BufIterator: buffer full")
-	}
-	itr.buf.full = true
-}
-
 // The maximum size of array containers.
 const ArrayMaxSize = 4096
 
@@ -2377,7 +2327,7 @@ func (c *container) bitmapZeroRange(i, j uint64) {
 
 func unionArrayBitmap(a, b *container) *container {
 	output := &container{}
-	itr := newBufIterator(newBitmapIterator(b.bitmap))
+	itr := newBufBitmapIterator(newBitmapIterator(b.bitmap))
 	for i := 0; ; {
 		vb, eof := itr.next()
 		if i >= len(a.array) && eof {
@@ -2549,7 +2499,7 @@ func differenceRunBitmap(a, b *container) *container {
 	if a.n == 0 || b.n == 0 {
 		return a.clone()
 	}
-	itr := newBufIterator(newBitmapIterator(b.bitmap))
+	itr := newBufBitmapIterator(newBitmapIterator(b.bitmap))
 	return differenceRunIterator(a, itr)
 }
 
@@ -2673,7 +2623,7 @@ func differenceRunRun(a, b *container) *container {
 
 func differenceArrayBitmap(a, b *container) *container {
 	output := &container{}
-	itr := newBufIterator(newBitmapIterator(b.bitmap))
+	itr := newBufBitmapIterator(newBitmapIterator(b.bitmap))
 	for i := 0; i < len(a.array); {
 		va := a.array[i]
 		vb, eof := itr.next()
@@ -2698,7 +2648,7 @@ func differenceArrayBitmap(a, b *container) *container {
 
 func differenceBitmapArray(a, b *container) *container {
 	output := &container{}
-	itr := newBufIterator(newBitmapIterator(a.bitmap))
+	itr := newBufBitmapIterator(newBitmapIterator(a.bitmap))
 	i := 0
 	va, eof := itr.next()
 	for {
@@ -2728,8 +2678,8 @@ func differenceBitmapArray(a, b *container) *container {
 
 func differenceBitmapBitmap(a, b *container) *container {
 	output := &container{}
-	itr0 := newBufIterator(newBitmapIterator(a.bitmap))
-	itr1 := newBufIterator(newBitmapIterator(b.bitmap))
+	itr0 := newBufBitmapIterator(newBitmapIterator(a.bitmap))
+	itr1 := newBufBitmapIterator(newBitmapIterator(b.bitmap))
 	v0, eof0 := itr0.next()
 	v1, eof1 := itr1.next()
 	for {
@@ -3028,7 +2978,7 @@ type containerIterator interface {
 	next() (uint32, bool)
 }
 
-// bitmapIterator represents an iterator over container array values.
+// arrayIterator represents an iterator over container array values.
 type arrayIterator struct {
 	array []uint32
 	i     int
@@ -3102,7 +3052,7 @@ type bufBitmapIterator struct {
 }
 
 // newBufBitmapIterator returns a buffered iterator that wraps a bitmapIterator.
-func newBufIterator(itr *bitmapIterator) *bufBitmapIterator {
+func newBufBitmapIterator(itr *bitmapIterator) *bufBitmapIterator {
 	return &bufBitmapIterator{itr: itr}
 }
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 3cb9742c5..75a686443 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -2159,3 +2159,62 @@ func TestBitmap_BitmapWriteToWithEmpty(t *testing.T) {
 		t.Fatalf("Counts do not match after a marshal %d %d", bm0.Count(), bm1.Count())
 	}
 }
+
+func Test_BufBitmapIterator_Next(t *testing.T) {
+	b := NewBitmap()
+	for i := uint64(0); i < 4097; i++ {
+		b.Add(i)
+	}
+	if !b.containers[0].isBitmap() {
+		t.Fatalf("wrong container type")
+	}
+
+	bin := []uint32{}
+
+	itr := newBufBitmapIterator(newBitmapIterator(b.containers[0].bitmap))
+	x := uint32(0)
+
+	for i := 0; i < 10; i++ {
+		x, _ = itr.next()
+		bin = append(bin, x)
+	}
+	exp := []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+	if !reflect.DeepEqual(bin, exp) {
+		t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin)
+	}
+
+	// ensure that unread points next back one such that the last value is repeated
+	itr.unread()
+	x, _ = itr.next()
+	bin = append(bin, x)
+	exp = append(exp, uint32(9))
+	if !reflect.DeepEqual(bin, exp) {
+		t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin)
+	}
+}
+
+func Test_BufBitmapIterator_UnreadPanic(t *testing.T) {
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Errorf("BufBitmapIterator unread did not panic")
+		}
+	}()
+
+	b := NewBitmap()
+	for i := uint64(0); i < 4097; i++ {
+		b.Add(i)
+	}
+	if !b.containers[0].isBitmap() {
+		t.Fatalf("wrong container type")
+	}
+
+	itr := newBufBitmapIterator(newBitmapIterator(b.containers[0].bitmap))
+	for i := 0; i < 10; i++ {
+		itr.next()
+	}
+
+	// ensure that unreading back-to-back panics
+	itr.unread()
+	itr.unread()
+}

From 230f419df438f29b6ef958fe79ad6c9e78bc7083 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <jaffee@umbel.com>
Date: Wed, 5 Jul 2017 11:10:17 -0500
Subject: [PATCH 089/100] fixes from code review - style/grammar/comments

---
 roaring/roaring.go | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 30b4732c1..810c378ef 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -38,8 +38,9 @@ const (
 	cookieNoRuns = magicNumberNoRuns + storageVersion<<16
 	cookie       = magicNumber + storageVersion<<16
 
-	// headerBaseSize is the size of the cookie and key count at the beginning of a file.
-	// Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8.
+	// headerBaseSize is the size in bytes of the cookie and key count at the
+	// beginning of a file. Headers in files with runs also include
+	// runFlagBitset, of length (numContainers+7)/8.
 	headerBaseSize = 4 + 4
 
 	// runCountHeaderSize is the size in bytes of the run count stored
@@ -341,7 +342,9 @@ func (b *Bitmap) insertAt(key uint64, c *container, i int) {
 	b.containers = insertContainer(b.containers, i, c)
 }
 
-// IntersectionCount returns the number of intersections between b and other.
+// IntersectionCount returns the number of set bits that would result in an
+// intersection between b and other. It is more efficient than actually
+// intersecting the two and counting the result.
 func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
 	var n uint64
 	for i, j := 0, 0; i < len(b.containers) && j < len(other.containers); {
@@ -689,7 +692,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 			opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval32Size
 		} else if c.n <= ArrayMaxSize { // Read array.
 			c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
-			// TODO: instead of commenting this out, we ne ed to make it a configuration option
+			// TODO: instead of commenting this out, we need to make it a configuration option
 			//for _, v := range c.array {
 			//    assert(lowbits(uint64(v)) == v, "array value out of range: %d", v)
 			//}
@@ -1016,12 +1019,12 @@ func (c *container) isArray() bool {
 	return c.bitmap == nil && c.runs == nil
 }
 
-// isBitmap returns true if the container is a bitmap container
+// isBitmap returns true if the container is a bitmap container.
 func (c *container) isBitmap() bool {
 	return c.array == nil && c.runs == nil
 }
 
-// isRun returns true if the container is a run-length-encoded container
+// isRun returns true if the container is a run-length-encoded container.
 func (c *container) isRun() bool {
 	return c.array == nil && c.bitmap == nil
 }
@@ -1270,6 +1273,8 @@ func (c *container) arrayCountRuns() (r int) {
 	return r
 }
 
+// Optimize converts the container to the type which will take up the least
+// amount of space.
 func (c *container) Optimize() {
 	if c.isArray() {
 		runs := c.arrayCountRuns()
@@ -1309,7 +1314,8 @@ func binSearchRuns(v uint32, a []interval32) (int, bool) {
 	return i, false
 }
 
-//runContains determines if v is in the containers run set.
+// runContains determines if v is in the container assuming c is a run
+// container.
 func (c *container) runContains(v uint32) bool {
 	_, found := binSearchRuns(v, c.runs)
 	return found
@@ -1724,7 +1730,7 @@ func (c *container) check() error {
 // ContainerInfo represents a point-in-time snapshot of container stats.
 type ContainerInfo struct {
 	Key     uint64         // container key
-	Type    string         // container type (array or bitmap)
+	Type    string         // container type (array, bitmap, or run)
 	N       int            // number of bits
 	Alloc   int            // memory used
 	Pointer unsafe.Pointer // offset within the mmap

From 2b27f36dac57e52cff52c895a26d14985e6e4e46 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Mon, 10 Jul 2017 09:02:21 -0500
Subject: [PATCH 090/100] added error logging to low level serve method

---
 server.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/server.go b/server.go
index 37fadedd5..452494429 100644
--- a/server.go
+++ b/server.go
@@ -162,7 +162,12 @@ func (s *Server) Open() error {
 	s.Holder.LogOutput = s.LogOutput
 
 	// Serve HTTP.
-	go func() { http.Serve(ln, s.Handler) }()
+	go func() {
+		err := http.Serve(ln, s.Handler)
+		if err != nil {
+			s.logger().Printf("HTTP handler terminated with error: %s\n", err)
+		}
+	}()
 
 	// Start background monitoring.
 	s.wg.Add(3)

From d7272f46a231d289a81b006b1a21e58e0ce875c4 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Thu, 27 Jul 2017 13:00:14 -0500
Subject: [PATCH 091/100] converted containers to 16bit

---
 roaring/internal_test.go         |  10 +-
 roaring/roaring.go               | 442 ++++++++--------
 roaring/roaring_internal_test.go | 838 +++++++++++++++----------------
 roaring/roaring_test.go          |  12 +-
 4 files changed, 637 insertions(+), 665 deletions(-)

diff --git a/roaring/internal_test.go b/roaring/internal_test.go
index 59fbccf1c..107851489 100644
--- a/roaring/internal_test.go
+++ b/roaring/internal_test.go
@@ -23,29 +23,29 @@ import (
 func TestBitmapIterator(t *testing.T) {
 	for i, tt := range []struct {
 		bitmap []uint64
-		values []uint32
+		values []uint16
 	}{
 		// Empty
 		{
 			bitmap: []uint64{6}, // 0110
-			values: []uint32{1, 2},
+			values: []uint16{1, 2},
 		},
 
 		// Single uint64 bitmap
 		{
 			bitmap: []uint64{6}, // 0110
-			values: []uint32{1, 2},
+			values: []uint16{1, 2},
 		},
 
 		// Multi uint64 bitmap
 		{
 			bitmap: []uint64{1 << 63, 1, 0, 1, 3 << 62},
-			values: []uint32{63, 64, 192, 318, 319},
+			values: []uint16{63, 64, 192, 318, 319},
 		},
 	} {
 		itr := newBitmapIterator(tt.bitmap)
 
-		var a []uint32
+		var a []uint16
 		for v, eof := itr.next(); !eof; v, eof = itr.next() {
 			a = append(a, v)
 		}
diff --git a/roaring/roaring.go b/roaring/roaring.go
index 810c378ef..4c43e000e 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -48,13 +48,17 @@ const (
 	runCountHeaderSize = 2
 
 	// interval32Size is the size of a single run in a container.runs.
-	interval32Size = 8
+	interval16Size = 4
 
 	// bitmapN is the number of values in a container.bitmap.
 	bitmapN = (1 << 16) / 64
 
 	// manual allocation size tuned to our average client data
-	manualAlloc = 524288
+	manualAlloc     = 524288
+	ContainerArray  = byte(1)
+	ContainerBitmap = byte(2)
+	ContainerRun    = byte(3)
+	maxContainerVal = 0xffff
 )
 
 // Bitmap represents a roaring bitmap.
@@ -126,10 +130,9 @@ func (b *Bitmap) add(v uint64) bool {
 	// If index is negative then there's not an exact match
 	// and a container needs to be added.
 	if i < 0 {
-		b.insertAt(hb, newContainer(), -i-1)
+		b.insertAt(hb, newContainer(), int(-i-1))
 		i = -i - 1
 	}
-
 	return b.containers[i].add(lowbits(v))
 }
 
@@ -193,19 +196,20 @@ func (b *Bitmap) Count() (n uint64) {
 
 // CountRange returns the number of bits set between [start, end).
 func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
+
 	i := search64(b.keys, highbits(start))
 	j := search64(b.keys, highbits(end))
 
 	// If range is entirely in one container then just count that range.
 	if i >= 0 && i == j {
-		return uint64(b.containers[i].countRange(lowbits(start), lowbits(end)))
+		return uint64(b.containers[i].countRange(int(lowbits(start)), int(lowbits(end))))
 	}
 
 	// Count first partial container.
 	if i < 0 {
 		i = -i
 	} else {
-		n += uint64(b.containers[i].countRange(lowbits(start), (bitmapN*64)+1))
+		n += uint64(b.containers[i].countRange(int(lowbits(start)), maxContainerVal+1))
 	}
 
 	// Count last container.
@@ -215,7 +219,7 @@ func (b *Bitmap) CountRange(start, end uint64) (n uint64) {
 			j = len(b.containers)
 		}
 	} else {
-		n += uint64(b.containers[j].countRange(0, lowbits(end)))
+		n += uint64(b.containers[j].countRange(0, int(lowbits(end))))
 	}
 
 	// Count containers in between.
@@ -310,36 +314,14 @@ func (b *Bitmap) container(key uint64) *container {
 	return b.containers[i]
 }
 
-func insertU64(original []uint64, position int, value uint64) []uint64 {
-	l := len(original)
-	target := original
-	if cap(original) == l {
-		target = make([]uint64, l+1, l+manualAlloc)
-		copy(target, original[:position])
-	} else {
-		target = append(target, 0)
-	}
-	copy(target[position+1:], original[position:])
-	target[position] = value
-	return target
-}
-
-func insertContainer(original []*container, position int, value *container) []*container {
-	l := len(original)
-	target := original
-	if cap(original) == l {
-		target = make([]*container, l+1, l+manualAlloc)
-		copy(target, original[:position])
-	} else {
-		target = append(target, nil)
-	}
-	copy(target[position+1:], original[position:])
-	target[position] = value
-	return target
-}
 func (b *Bitmap) insertAt(key uint64, c *container, i int) {
-	b.keys = insertU64(b.keys, i, key)
-	b.containers = insertContainer(b.containers, i, c)
+	b.keys = append(b.keys, 0)
+	copy(b.keys[i+1:], b.keys[i:])
+	b.keys[i] = key
+
+	b.containers = append(b.containers, nil)
+	copy(b.containers[i+1:], b.containers[i:])
+	b.containers[i] = c
 }
 
 // IntersectionCount returns the number of set bits that would result in an
@@ -528,6 +510,15 @@ func (b *Bitmap) Optimize() {
 		c.Optimize()
 	}
 }
+func checkA(a []uint16) bool {
+	for _, x := range a {
+		if x > 65535 {
+			return true
+		}
+	}
+	return false
+
+}
 
 // WriteTo writes b to w.
 func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
@@ -536,43 +527,17 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 	//b.removeEmptyContainers()
 	containerCount := len(b.keys) - b.countEmptyContainers()
 
-	// Create bitset indicating runs, record whether any runs present.
-	containsRuns := false
-	runFlagBitset := make([]uint8, (containerCount+7)/8)
-	k := 0
-	for _, c := range b.containers {
-		if c.n == 0 {
-			continue
-		}
-		if c.isRun() {
-			containsRuns = true
-			runFlagBitset[k/8] |= (1 << uint(k%8))
-		}
-		k++
-	}
-
 	thisCookie := cookieNoRuns
 	headerSize := headerBaseSize
-	if containsRuns {
-		thisCookie = cookie
-		headerSize += len(runFlagBitset)
-	}
 
 	// Build header before writing individual container blocks.
-	// Metadata for each container is 4+8+4 = sizeof(key) + sizeof(cardinality) + sizeof(file offset)
-	buf := make([]byte, headerSize+(containerCount*(4+8+4)))
+	// Metadata for each container is 4+8+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset)
+	buf := make([]byte, headerSize+(containerCount*(4+4+4+4)))
 
 	// Cookie header section.
 	binary.LittleEndian.PutUint32(buf[0:], thisCookie)
 	binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount))
 
-	if containsRuns {
-		// Write runFlag bitset.
-		for i, b := range runFlagBitset {
-			buf[8+i] = b
-		}
-	}
-
 	empty := 0
 	// Descriptive header section: encode keys and cardinality.
 	// Key and cardinality are stored interleaved here, 12 bytes per container.
@@ -585,7 +550,8 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 		//assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n)
 		if c.n > 0 {
 			binary.LittleEndian.PutUint64(buf[headerSize+(i-empty)*12:], uint64(key))
-			binary.LittleEndian.PutUint32(buf[headerSize+(i-empty)*12+8:], uint32(c.n-1))
+			buf[headerSize+(i-empty)*12+8] = c.container_type
+			binary.LittleEndian.PutUint16(buf[headerSize+(i-empty)*12+8+2:], uint16(c.n-1))
 		} else {
 			empty++
 		}
@@ -635,11 +601,10 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 	// Verify the first two bytes are a valid magicNumber, and second two bytes match current storageVersion.
 	fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
 	fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4]))
-	containsRuns := false
 	if fileMagic == magicNumberNoRuns {
 		// noop
-	} else if fileMagic == magicNumber {
-		containsRuns = true
+		//	} else if fileMagic == magicNumber {
+		//	containsRuns = true
 	} else {
 		return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic)
 	}
@@ -655,21 +620,13 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 
 	headerSize := headerBaseSize
 
-	runFlagBitset := make([]uint8, (keyN+7)/8)
-	if containsRuns {
-		// Read runFlag bitset.
-		for i := 0; i < len(runFlagBitset); i++ {
-			runFlagBitset[i] = data[8+i]
-		}
-		headerSize += len(runFlagBitset)
-	}
-
 	// Descriptive header section: Read container keys and cardinalities.
 	for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
 		b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
 		b.containers[i] = &container{
-			n:      int(binary.LittleEndian.Uint32(buf[8:12])) + 1,
-			mapped: true,
+			container_type: buf[8],
+			n:              int(binary.LittleEndian.Uint16(buf[10:12]) + 1),
+			mapped:         true,
 		}
 	}
 	opsOffset := headerSize + int(keyN)*12
@@ -685,26 +642,18 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 
 		// Map byte slice directly to the container data.
 		c := b.containers[i]
-
-		if containsRuns && (runFlagBitset[i/8]&(1<<uint(i%8))) != 0 { // Read runs.
+		switch c.container_type {
+		case ContainerRun:
 			runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
-			c.runs = (*[0xFFFFFFF]interval32)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount]
-			opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval32Size
-		} else if c.n <= ArrayMaxSize { // Read array.
-			c.array = (*[0xFFFFFFF]uint32)(unsafe.Pointer(&data[offset]))[:c.n]
-			// TODO: instead of commenting this out, we need to make it a configuration option
-			//for _, v := range c.array {
-			//    assert(lowbits(uint64(v)) == v, "array value out of range: %d", v)
-			//}
-			opsOffset = int(offset) + len(c.array)*4 // sizeof(uint32)
-		} else { // Read bitmap.
+			c.runs = (*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount]
+			opsOffset = int(offset) + runCountHeaderSize + len(c.runs)*interval16Size
+		case ContainerArray:
+			c.array = (*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.n]
+			opsOffset = int(offset) + len(c.array)*2 // sizeof(uint32)
+		case ContainerBitmap:
 			c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
 			opsOffset = int(offset) + len(c.bitmap)*8 // sizeof(uint64)
 		}
-		// Verify container count on load.
-		// TODO: instead of commenting this out, we need to make it a configuration option
-		//count := c.count()
-		//assert(c.count() == c.n, "container count mismatch: count=%d, n=%d", count, c.n)
 	}
 
 	// Read ops log until the end of the file.
@@ -834,7 +783,7 @@ type Iterator struct {
 }
 
 // eof returns true if the iterator is at the end of the bitmap.
-func (itr *Iterator) eof() bool { return itr.i >= len(itr.bitmap.containers) }
+func (itr *Iterator) eof() bool { return int(itr.i) >= len(itr.bitmap.containers) }
 
 // Seek moves to the first value equal to or greater than `seek`.
 func (itr *Iterator) Seek(seek uint64) {
@@ -849,6 +798,9 @@ func (itr *Iterator) Seek(seek uint64) {
 
 	// Move to the correct value index inside the container.
 	lb := lowbits(seek)
+	if int(itr.i) >= len(itr.bitmap.containers) {
+		panic(fmt.Sprintf("data Corruption %d %d %d", itr.i, len(itr.bitmap.containers), seek))
+	}
 	c := itr.bitmap.containers[itr.i]
 	if c.isArray() {
 		// Find index in the container.
@@ -856,7 +808,7 @@ func (itr *Iterator) Seek(seek uint64) {
 		if itr.j < 0 {
 			itr.j = -itr.j - 1
 		}
-		if itr.j < len(c.array) {
+		if int(itr.j) < len(c.array) {
 			itr.j--
 			return
 		}
@@ -899,7 +851,7 @@ func (itr *Iterator) Next() (v uint64, eof bool) {
 
 		c := itr.bitmap.containers[itr.i]
 		if c.isArray() {
-			if itr.j >= c.n-1 {
+			if itr.j >= int(c.n-1) {
 				// Reached end of array, move to the next container.
 				itr.i, itr.j = itr.i+1, -1
 				continue
@@ -974,7 +926,7 @@ func (itr *Iterator) peek() uint64 {
 		return uint64(key)<<16 | uint64(c.array[itr.j])
 	}
 	if c.isRun() {
-		return uint64(key)<<16 | uint64(c.runs[itr.j].start+uint32(itr.k))
+		return uint64(key)<<16 | uint64(c.runs[itr.j].start+uint16(itr.k))
 	}
 	return uint64(key)<<16 | uint64(itr.j)
 }
@@ -992,26 +944,27 @@ const RunMaxSize = 2048
 // an array or RLE container is used, depending on the contents. For containers
 // with more than 4,096 values, the values are encoded into bitmaps.
 type container struct {
-	n      int          // number of integers in container
-	array  []uint32     // used for array containers
-	bitmap []uint64     // used for bitmap containers
-	runs   []interval32 // used for RLE containers
-	mapped bool         // mapped directly to a byte slice when true
+	container_type byte         // number of integers in container
+	n              int          // number of integers in container
+	array          []uint16     // used for array containers
+	bitmap         []uint64     // used for bitmap containers
+	runs           []interval16 // used for RLE containers
+	mapped         bool         // mapped directly to a byte slice when true
 }
 
-type interval32 struct {
-	start uint32
-	last  uint32
+type interval16 struct {
+	start uint16
+	last  uint16
 }
 
 // runlen returns the count of integers in the interval.
-func (iv interval32) runlen() int {
-	return 1 + int(iv.last) - int(iv.start)
+func (iv interval16) runlen() int {
+	return int(1 + iv.last - iv.start)
 }
 
 // newContainer returns a new instance of container.
 func newContainer() *container {
-	return &container{}
+	return &container{container_type: ContainerArray}
 }
 
 // isArray returns true if the container is an array container.
@@ -1039,7 +992,7 @@ func (c *container) unmap() {
 	}
 
 	if c.array != nil {
-		tmp := make([]uint32, len(c.array))
+		tmp := make([]uint16, len(c.array))
 		copy(tmp, c.array)
 		c.array = tmp
 	}
@@ -1049,7 +1002,7 @@ func (c *container) unmap() {
 		c.bitmap = tmp
 	}
 	if c.runs != nil {
-		tmp := make([]interval32, len(c.runs))
+		tmp := make([]interval16, len(c.runs))
 		copy(tmp, c.runs)
 		c.runs = tmp
 	}
@@ -1058,11 +1011,11 @@ func (c *container) unmap() {
 
 // count counts all bits in the container.
 func (c *container) count() (n int) {
-	return c.countRange(0, (bitmapN*64)+1)
+	return c.countRange(0, maxContainerVal+1)
 }
 
 // countRange counts the number of bits set between [start, end).
-func (c *container) countRange(start, end uint32) (n int) {
+func (c *container) countRange(start, end int) (n int) {
 	if c.isArray() {
 		return c.arrayCountRange(start, end)
 	} else if c.isRun() {
@@ -1071,10 +1024,10 @@ func (c *container) countRange(start, end uint32) (n int) {
 	return c.bitmapCountRange(start, end)
 }
 
-func (c *container) arrayCountRange(start, end uint32) (n int) {
-	i := sort.Search(len(c.array), func(i int) bool { return c.array[i] >= start })
+func (c *container) arrayCountRange(start, end int) (n int) {
+	i := sort.Search(len(c.array), func(i int) bool { return int(c.array[i]) >= start })
 	for ; i < len(c.array); i++ {
-		v := c.array[i]
+		v := int(c.array[i])
 		if v >= end {
 			break
 		}
@@ -1083,19 +1036,18 @@ func (c *container) arrayCountRange(start, end uint32) (n int) {
 	return n
 }
 
-func (c *container) bitmapCountRange(start, end uint32) int {
+func (c *container) bitmapCountRange(start, end int) int {
 	var n uint64
 	i, j := start/64, end/64
-
 	// Special case when start and end fall in the same word.
 	if i == j {
-		offi, offj := start%64, 64-end%64
+		offi, offj := uint(start%64), uint(64-end%64)
 		n += popcount((c.bitmap[i] >> offi) << (offj + offi))
 		return int(n)
 	}
 
 	// Count partial starting word.
-	if off := start % 64; off != 0 {
+	if off := uint(start) % 64; off != 0 {
 		n += popcount(c.bitmap[i] >> off)
 		i++
 	}
@@ -1107,45 +1059,46 @@ func (c *container) bitmapCountRange(start, end uint32) int {
 
 	// Count partial ending word.
 	if int(j) < len(c.bitmap) {
-		off := 64 - (end % 64)
+		off := 64 - (uint(end) % 64)
 		n += popcount(c.bitmap[j] << off)
 	}
 
 	return int(n)
 }
 
-func (c *container) runCountRange(start, end uint32) (n int) {
+func (c *container) runCountRange(start, end int) (n int) {
 	for _, iv := range c.runs {
 		// iv is before range
-		if iv.last < start {
+		if int(iv.last) < start {
 			continue
 		}
 		// iv is after range
-		if end < iv.start {
+		if end < int(iv.start) {
 			break
 		}
 		// iv is superset of range
-		if iv.start < start && iv.last > end {
+		if int(iv.start) < start && int(iv.last) > end {
 			return int(end - start)
 		}
 		// iv is subset of range
-		if iv.start >= start && iv.last < end {
+		if int(iv.start) >= start && int(iv.last) < end {
 			n += iv.runlen()
 		}
 		// iv overlaps beginning of range
-		if iv.start < start && iv.last < end {
-			n += int(iv.last - start + 1)
+		if int(iv.start) < start && int(iv.last) < end {
+			n += int(iv.last) - start + 1
 		}
 		// iv overlaps end of range
-		if iv.start > start && iv.last >= end {
-			n += int(end - iv.start)
+		if int(iv.start) > start && int(iv.last) >= end {
+			n += end - int(iv.start)
 		}
 	}
 	return n
 }
 
 // add adds a value to the container.
-func (c *container) add(v uint32) (added bool) {
+func (c *container) add(v uint16) (added bool) {
+
 	if c.isArray() {
 		added = c.arrayAdd(v)
 	} else if c.isRun() {
@@ -1159,7 +1112,7 @@ func (c *container) add(v uint32) (added bool) {
 	return added
 }
 
-func (c *container) arrayAdd(v uint32) bool {
+func (c *container) arrayAdd(v uint16) bool {
 	// Optimize appending to the end of an array container.
 	if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && c.array[c.n-1] < v {
 		c.unmap()
@@ -1189,7 +1142,7 @@ func (c *container) arrayAdd(v uint32) bool {
 
 }
 
-func (c *container) bitmapAdd(v uint32) bool {
+func (c *container) bitmapAdd(v uint16) bool {
 	if c.bitmapContains(v) {
 		return false
 	}
@@ -1198,14 +1151,14 @@ func (c *container) bitmapAdd(v uint32) bool {
 	return true
 }
 
-func (c *container) runAdd(v uint32) bool {
+func (c *container) runAdd(v uint16) bool {
 	if len(c.runs) == 0 {
 		c.unmap()
-		c.runs = []interval32{{start: v, last: v}}
+		c.runs = []interval16{{start: v, last: v}}
 		return true
 	}
 	i := 0
-	var iv interval32
+	var iv interval16
 	for i, iv = range c.runs {
 		if iv.last >= v {
 			break
@@ -1219,7 +1172,7 @@ func (c *container) runAdd(v uint32) bool {
 		if iv.last == v-1 {
 			c.runs[i].last += 1
 		} else {
-			c.runs = append(c.runs, interval32{start: v, last: v})
+			c.runs = append(c.runs, interval16{start: v, last: v})
 		}
 	} else if v+1 == iv.start {
 		// combining two intervals
@@ -1235,14 +1188,14 @@ func (c *container) runAdd(v uint32) bool {
 		c.runs[i-1].last += 1
 	} else {
 		// alone
-		newIv := interval32{start: v, last: v}
-		c.runs = append(c.runs[:i], append([]interval32{newIv}, c.runs[i:]...)...)
+		newIv := interval16{start: v, last: v}
+		c.runs = append(c.runs[:i], append([]interval16{newIv}, c.runs[i:]...)...)
 	}
 	return true
 }
 
 // contains returns true if v is in the container.
-func (c *container) contains(v uint32) bool {
+func (c *container) contains(v uint16) bool {
 	if c.isArray() {
 		return c.arrayContains(v)
 	} else if c.isRun() {
@@ -1265,7 +1218,7 @@ func (c *container) bitmapCountRuns() (r int) {
 func (c *container) arrayCountRuns() (r int) {
 	prev := -2
 	for _, v := range c.array {
-		if uint32(prev+1) != v {
+		if prev+1 != int(v) {
 			r += 1
 		}
 		prev = int(v)
@@ -1294,17 +1247,17 @@ func (c *container) Optimize() {
 
 }
 
-func (c *container) arrayContains(v uint32) bool {
+func (c *container) arrayContains(v uint16) bool {
 	return search32(c.array, v) >= 0
 }
 
-func (c *container) bitmapContains(v uint32) bool {
+func (c *container) bitmapContains(v uint16) bool {
 	return (c.bitmap[v/64] & (1 << uint64(v%64))) != 0
 }
 
 // binSearchRuns returns the index of the run containing v, and true, when v is contained;
 // or the index of the next run starting after v, and false, when v is not contained.
-func binSearchRuns(v uint32, a []interval32) (int, bool) {
+func binSearchRuns(v uint16, a []interval16) (int, bool) {
 	i := sort.Search(len(a),
 		func(i int) bool { return a[i].last >= v })
 	if i < len(a) {
@@ -1316,13 +1269,13 @@ func binSearchRuns(v uint32, a []interval32) (int, bool) {
 
 // runContains determines if v is in the container assuming c is a run
 // container.
-func (c *container) runContains(v uint32) bool {
+func (c *container) runContains(v uint16) bool {
 	_, found := binSearchRuns(v, c.runs)
 	return found
 }
 
 // remove removes a value from the container.
-func (c *container) remove(v uint32) (removed bool) {
+func (c *container) remove(v uint16) (removed bool) {
 	if c.isArray() {
 		removed = c.arrayRemove(v)
 	} else if c.isRun() {
@@ -1336,7 +1289,7 @@ func (c *container) remove(v uint32) (removed bool) {
 	return removed
 }
 
-func (c *container) arrayRemove(v uint32) bool {
+func (c *container) arrayRemove(v uint16) bool {
 	i := search32(c.array, v)
 	if i < 0 {
 		return false
@@ -1347,7 +1300,7 @@ func (c *container) arrayRemove(v uint32) bool {
 	return true
 }
 
-func (c *container) bitmapRemove(v uint32) bool {
+func (c *container) bitmapRemove(v uint16) bool {
 	if !c.bitmapContains(v) {
 		return false
 	}
@@ -1365,7 +1318,7 @@ func (c *container) bitmapRemove(v uint32) bool {
 }
 
 // runRemove removes v from a run container, and returns true if v was removed.
-func (c *container) runRemove(v uint32) bool {
+func (c *container) runRemove(v uint16) bool {
 	i, contains := binSearchRuns(v, c.runs)
 	if !contains {
 		return false
@@ -1380,13 +1333,13 @@ func (c *container) runRemove(v uint32) bool {
 	} else if v > c.runs[i].start {
 		last := c.runs[i].last
 		c.runs[i].last = v - 1
-		c.runs = append(c.runs[:i+1], append([]interval32{{start: v + 1, last: last}}, c.runs[i+1:]...)...)
+		c.runs = append(c.runs[:i+1], append([]interval16{{start: v + 1, last: last}}, c.runs[i+1:]...)...)
 	}
 	return true
 }
 
 // max returns the maximum value in the container.
-func (c *container) max() uint32 {
+func (c *container) max() uint16 {
 	if c.isArray() {
 		return c.arrayMax()
 	} else if c.isRun() {
@@ -1396,14 +1349,14 @@ func (c *container) max() uint32 {
 	}
 }
 
-func (c *container) arrayMax() uint32 {
+func (c *container) arrayMax() uint16 {
 	if len(c.array) == 0 {
 		return 0 // probably hiding some ugly bug but it prevents a crash
 	}
 	return c.array[len(c.array)-1]
 }
 
-func (c *container) bitmapMax() uint32 {
+func (c *container) bitmapMax() uint16 {
 	// Search bitmap in reverse order.
 	for i := len(c.bitmap) - 1; i >= 0; i-- {
 		// If value is zero then skip.
@@ -1413,25 +1366,26 @@ func (c *container) bitmapMax() uint32 {
 		}
 
 		// Find the highest set bit.
-		for j := uint32(63); j >= 0; j-- {
+		for j := uint16(63); j >= 0; j-- {
 			if v&(1<<j) != 0 {
-				return uint32(i)*64 + j
+				return uint16(i)*64 + j
 			}
 		}
 	}
 	return 0
 }
 
-func (c *container) runMax() uint32 {
+func (c *container) runMax() uint16 {
 	if len(c.runs) == 0 {
 		return 0
 	}
-	return uint32(c.runs[len(c.runs)-1].last)
+	return uint16(c.runs[len(c.runs)-1].last)
 }
 
 // bitmapToArray converts from bitmap format to array format.
 func (c *container) bitmapToArray() {
-	c.array = make([]uint32, 0, c.n)
+	c.array = make([]uint16, 0, c.n)
+	c.container_type = ContainerArray
 
 	// return early if empty
 	if c.n == 0 {
@@ -1443,7 +1397,7 @@ func (c *container) bitmapToArray() {
 	for i, bitmap := range c.bitmap {
 		for bitmap != 0 {
 			t := bitmap & -bitmap
-			c.array = append(c.array, uint32((i*64 + int(popcount(t-1)))))
+			c.array = append(c.array, uint16((i*64 + int(popcount(t-1)))))
 			bitmap ^= t
 		}
 	}
@@ -1454,6 +1408,7 @@ func (c *container) bitmapToArray() {
 // arrayToBitmap converts from array format to bitmap format.
 func (c *container) arrayToBitmap() {
 	c.bitmap = make([]uint64, bitmapN)
+	c.container_type = ContainerBitmap
 
 	// return early if empty
 	if c.n == 0 {
@@ -1472,6 +1427,7 @@ func (c *container) arrayToBitmap() {
 // runToBitmap converts from RLE format to bitmap format.
 func (c *container) runToBitmap() {
 	c.bitmap = make([]uint64, bitmapN)
+	c.container_type = ContainerBitmap
 
 	// return early if empty
 	if c.n == 0 {
@@ -1492,19 +1448,20 @@ func (c *container) runToBitmap() {
 
 // bitmapToRun converts from bitmap format to RLE format.
 func (c *container) bitmapToRun() {
+	c.container_type = ContainerRun
 	// return early if empty
 	if c.n == 0 {
-		c.runs = make([]interval32, 0)
+		c.runs = make([]interval16, 0)
 		c.bitmap = nil
 		c.mapped = false
 		return
 	}
 
 	numRuns := c.bitmapCountRuns()
-	c.runs = make([]interval32, 0, numRuns)
+	c.runs = make([]interval16, 0, numRuns)
 
 	current := c.bitmap[0]
-	var i, start, last uint32
+	var i, start, last uint16
 	for {
 		// skip while empty
 		for current == 0 && i < bitmapN-1 {
@@ -1515,7 +1472,7 @@ func (c *container) bitmapToRun() {
 		if current == 0 {
 			break
 		}
-		currentStart := uint32(trailingZeroN(current))
+		currentStart := uint16(trailingZeroN(current))
 		start = 64*i + currentStart
 
 		// pad LSBs with 1s
@@ -1529,12 +1486,12 @@ func (c *container) bitmapToRun() {
 
 		if current == maxBitmap {
 			// bitmap[1023] == maxBitmap
-			c.runs = append(c.runs, interval32{start, 65535})
+			c.runs = append(c.runs, interval16{start, maxContainerVal})
 			break
 		}
-		currentLast := uint32(trailingZeroN(^current))
+		currentLast := uint16(trailingZeroN(^current))
 		last = 64*i + currentLast
-		c.runs = append(c.runs, interval32{start, last - 1})
+		c.runs = append(c.runs, interval16{start, last - 1})
 
 		// pad LSBs with 0s
 		current = current & (current + 1)
@@ -1546,33 +1503,35 @@ func (c *container) bitmapToRun() {
 
 // arrayToRun converts from array format to RLE format.
 func (c *container) arrayToRun() {
+	c.container_type = ContainerRun
 	// return early if empty
 	if c.n == 0 {
-		c.runs = make([]interval32, 0)
+		c.runs = make([]interval16, 0)
 		c.array = nil
 		c.mapped = false
 		return
 	}
 
 	numRuns := c.arrayCountRuns()
-	c.runs = make([]interval32, 0, numRuns)
+	c.runs = make([]interval16, 0, numRuns)
 	start := c.array[0]
 	for i, v := range c.array[1:] {
 		if v-c.array[i] > 1 {
 			// if current-previous > 1, one run ends and another begins
-			c.runs = append(c.runs, interval32{start, c.array[i]})
+			c.runs = append(c.runs, interval16{start, c.array[i]})
 			start = v
 		}
 	}
 	// append final run
-	c.runs = append(c.runs, interval32{start, c.array[c.n-1]})
+	c.runs = append(c.runs, interval16{start, c.array[c.n-1]})
 	c.array = nil
 	c.mapped = false
 }
 
 // runToArray converts from RLE format to array format.
 func (c *container) runToArray() {
-	c.array = make([]uint32, 0, c.n)
+	c.container_type = ContainerArray
+	c.array = make([]uint16, 0, c.n)
 
 	// return early if empty
 	if c.n == 0 {
@@ -1592,10 +1551,10 @@ func (c *container) runToArray() {
 
 // clone returns a copy of c.
 func (c *container) clone() *container {
-	other := &container{n: c.n}
+	other := &container{n: c.n, container_type: c.container_type}
 
 	if c.array != nil {
-		other.array = make([]uint32, len(c.array))
+		other.array = make([]uint16, len(c.array))
 		copy(other.array, c.array)
 	}
 
@@ -1605,7 +1564,7 @@ func (c *container) clone() *container {
 	}
 
 	if c.runs != nil {
-		other.runs = make([]interval32, len(c.runs))
+		other.runs = make([]interval16, len(c.runs))
 		copy(other.runs, c.runs)
 	}
 
@@ -1630,12 +1589,12 @@ func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) {
 
 	// Verify all elements are valid.
 	// TODO: instead of commenting this out, we need to make it a configuration option
-	//for _, v := range c.array {
-	//  	assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v)
-	//}
+	for _, v := range c.array {
+		assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v)
+	}
 
 	// Write sizeof(uint32) * cardinality bytes.
-	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:4*c.n])
+	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:2*c.n])
 	return int64(nn), err
 }
 
@@ -1649,21 +1608,21 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) {
 	if len(c.runs) == 0 {
 		return 0, nil
 	}
-	// Write sizeof(interval32) * runCount bytes.
+	// Write sizeof(interval16) * runCount bytes.
 	err = binary.Write(w, binary.LittleEndian, uint16(len(c.runs)))
 	if err != nil {
 		return 0, err
 	}
-	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:interval32Size*len(c.runs)])
+	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:interval16Size*len(c.runs)])
 	return int64(runCountHeaderSize + nn), err
 }
 
 // size returns the encoded size of the container, in bytes.
 func (c *container) size() int {
 	if c.isArray() {
-		return len(c.array) * 4 // sizeof(uint32)
+		return len(c.array) * 2 // sizeof(uint16)
 	} else if c.isRun() {
-		return len(c.runs)*interval32Size + runCountHeaderSize
+		return len(c.runs)*interval16Size + runCountHeaderSize
 	} else {
 		return len(c.bitmap) * 8 // sizeof(uint64)
 	}
@@ -1675,10 +1634,10 @@ func (c *container) info() ContainerInfo {
 
 	if c.isArray() {
 		info.Type = "array"
-		info.Alloc = len(c.array) * 4 // sizeof(uint32)
+		info.Alloc = len(c.array) * 2 // sizeof(uint16)
 	} else if c.isRun() {
 		info.Type = "run"
-		info.Alloc = len(c.runs)*interval32Size + runCountHeaderSize
+		info.Alloc = len(c.runs)*interval16Size + runCountHeaderSize
 	} else {
 		info.Type = "bitmap"
 		info.Alloc = len(c.bitmap) * 8 // sizeof(uint64)
@@ -1702,16 +1661,16 @@ func (c *container) check() error {
 	var a ErrorList
 
 	if c.isArray() {
-		if len(c.array) != c.n {
+		if len(c.array) != int(c.n) {
 			a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(c.array), c.n))
 		}
 	} else if c.isRun() {
-		runCount := c.runCountRange(0, 0xFFFFFFFF)
-		if runCount != c.n {
-			a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", runCount, c.n))
+		n := c.runCountRange(0, maxContainerVal+1)
+		if n != c.n {
+			a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.n))
 		}
 	} else if c.isBitmap() {
-		if n := c.bitmapCountRange(0, uint32(len(c.bitmap)*64)); n != c.n {
+		if n := c.bitmapCountRange(0, maxContainerVal+1); n != c.n {
 			a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.n))
 		}
 	} else {
@@ -1797,7 +1756,7 @@ func intersectionCountArrayRun(a, b *container) (n uint64) {
 }
 
 func intersectionCountRunRun(a, b *container) uint64 {
-	var n uint32
+	var n uint16
 	na, nb := len(a.runs), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.runs[i], b.runs[j]
@@ -1830,7 +1789,7 @@ func intersectionCountRunRun(a, b *container) uint64 {
 
 func intersectionCountBitmapRun(a, b *container) (n uint64) {
 	for _, iv := range b.runs {
-		n += uint64(a.bitmapCountRange(iv.start, iv.last+1))
+		n += uint64(a.bitmapCountRange(int(iv.start), int(iv.last)+1))
 	}
 	return n
 }
@@ -1843,7 +1802,7 @@ func intersectionCountArrayBitmapOld(a, b *container) (n uint64) {
 	}
 
 	// Iterate over bitmap and find matching bits.
-	for i, bn := uint32(0), uint32(len(b.bitmap)); i < bn; i++ {
+	for i, bn := uint16(0), uint16(len(b.bitmap)); i < bn; i++ {
 		v := b.bitmap[i]
 
 		// Ignore if bytes are empty or array is done.
@@ -1852,7 +1811,7 @@ func intersectionCountArrayBitmapOld(a, b *container) (n uint64) {
 		}
 
 		// Check each bit.
-		for j := uint32(0); j < 64; j++ {
+		for j := uint16(0); j < 64; j++ {
 			if v&(1<<j) == 0 {
 				continue
 			}
@@ -1880,7 +1839,7 @@ func intersectionCountArrayBitmapOld(a, b *container) (n uint64) {
 func intersectionCountArrayBitmap(a, b *container) (n uint64) {
 	for _, val := range a.array {
 		i := val >> 6
-		if i >= uint32(len(b.bitmap)) {
+		if i >= uint16(len(b.bitmap)) {
 			break
 		}
 		off := val % 64
@@ -1974,7 +1933,7 @@ func intersectRunRun(a, b *container) *container {
 			j++
 		} else if va.last > vb.last && va.start >= vb.start {
 			// |--vb-|-|-va--|
-			output.n += output.runAppendInterval(interval32{start: va.start, last: vb.last})
+			output.n += output.runAppendInterval(interval16{start: va.start, last: vb.last})
 			j++
 		} else if va.last > vb.last && va.start < vb.start {
 			// |--va|--vb--|--|
@@ -1986,7 +1945,7 @@ func intersectRunRun(a, b *container) *container {
 			i++
 		} else if va.last <= vb.last && va.start < vb.start {
 			// |--va-|-|-vb--|
-			output.n += output.runAppendInterval(interval32{start: vb.start, last: va.last})
+			output.n += output.runAppendInterval(interval16{start: vb.start, last: va.last})
 			i++
 		}
 	}
@@ -2168,13 +2127,13 @@ func unionArrayArray(a, b *container) *container {
 // unionArrayRun optimistically assumes that the result will be a run container,
 // and converts to a bitmap or array container afterwards if necessary.
 func unionArrayRun(a, b *container) *container {
-	if b.n == 65536 {
+	if b.n == 65535 {
 		return b.clone()
 	}
 	output := &container{}
 	na, nb := len(a.array), len(b.runs)
-	var vb interval32
-	var va uint32
+	var vb interval16
+	var va uint16
 	for i, j := 0, 0; i < na || j < nb; {
 		if i < na {
 			va = a.array[i]
@@ -2183,7 +2142,7 @@ func unionArrayRun(a, b *container) *container {
 			vb = b.runs[j]
 		}
 		if i < na && (j >= nb || va < vb.start) {
-			output.n += output.runAppendInterval(interval32{start: va, last: va})
+			output.n += output.runAppendInterval(interval16{start: va, last: va})
 			i++
 		} else {
 			output.n += output.runAppendInterval(vb)
@@ -2204,12 +2163,15 @@ func unionArrayRun(a, b *container) *container {
 // interval is earlier than the start of the last interval in the list of runs.
 // Its return value is the amount by which the cardinality of the container was
 // increased.
-func (c *container) runAppendInterval(v interval32) int {
+func (c *container) runAppendInterval(v interval16) int {
 	if len(c.runs) == 0 {
 		c.runs = append(c.runs, v)
 		return int(v.last - v.start + 1)
 	} else {
 		last := c.runs[len(c.runs)-1]
+		if last.last == 65535 { //protect against overflow
+			return 0
+		}
 		if last.last+1 >= v.start && v.last > last.last {
 			c.runs[len(c.runs)-1].last = v.last
 			return int(v.last - last.last)
@@ -2222,17 +2184,17 @@ func (c *container) runAppendInterval(v interval32) int {
 }
 
 func unionRunRun(a, b *container) *container {
-	if a.n == 65536 {
+	if a.n == 65535 {
 		return a.clone()
 	}
-	if b.n == 65536 {
+	if b.n == 65535 {
 		return b.clone()
 	}
 	na, nb := len(a.runs), len(b.runs)
 	output := &container{
-		runs: make([]interval32, 0, na+nb),
+		runs: make([]interval16, 0, na+nb),
 	}
-	var va, vb interval32
+	var va, vb interval16
 	for i, j := 0, 0; i < na || j < nb; {
 		if i < na {
 			va = a.runs[i]
@@ -2255,7 +2217,7 @@ func unionRunRun(a, b *container) *container {
 }
 
 func unionBitmapRun(a, b *container) *container {
-	if b.n == 65536 {
+	if b.n == 65535 {
 		return b.clone()
 	}
 	output := a.clone()
@@ -2442,14 +2404,14 @@ func differenceArrayRun(a, b *container) *container {
 		return a.clone()
 	}
 
-	output := &container{array: make([]uint32, 0, a.n)}
+	output := &container{array: make([]uint16, 0, a.n)}
 	// cardinality upper bound: card(A)
 
 	i := 0 // array index
 	j := 0 // run index
 
 	// handle overlap
-	for i < a.n {
+	for i < int(a.n) {
 
 		// keep all array elements before beginning of runs
 		if a.array[i] < b.runs[j].start {
@@ -2478,7 +2440,7 @@ func differenceArrayRun(a, b *container) *container {
 		output.array = append(output.array, a.array[i:]...)
 		// TODO: consider handling container.n mutations in one place
 		// like we do with container.add().
-		output.n += len(a.array[i:])
+		output.n += int(len(a.array[i:]))
 	}
 	return output
 }
@@ -2516,7 +2478,7 @@ func differenceRunBitmap(a, b *container) *container {
 
 func differenceRunIterator(a *container, itr containerIterator) *container {
 
-	output := &container{runs: make([]interval32, 0, a.n)}
+	output := &container{runs: make([]interval16, 0, a.n)}
 
 	vb, eof := itr.next()
 	j := 0
@@ -2549,7 +2511,7 @@ func differenceRunIterator(a *container, itr containerIterator) *container {
 				working = false
 			}
 		case vb > vr.start: //inside run
-			output.n += output.runAppendInterval(interval32{start: vr.start, last: vb - 1})
+			output.n += output.runAppendInterval(interval16{start: vr.start, last: vb - 1})
 			vr.start = vb + 1
 
 		}
@@ -2584,7 +2546,7 @@ func differenceRunRun(a, b *container) *container {
 	alen := len(a.runs)
 	blen := len(b.runs)
 
-	output := &container{runs: make([]interval32, 0, alen+blen)} // TODO allocate max then truncate? or something else
+	output := &container{runs: make([]interval16, 0, alen+blen)} // TODO allocate max then truncate? or something else
 	// cardinality upper bound: sum of number of runs
 	// each B-run could split an A-run in two, up to len(b.runs) times
 
@@ -2592,7 +2554,7 @@ func differenceRunRun(a, b *container) *container {
 		switch {
 		case alast < bstart:
 			// current A-run entirely preceeds current B-run: keep full A-run, advance to next A-run
-			output.runs = append(output.runs, interval32{start: uint32(astart), last: uint32(alast)})
+			output.runs = append(output.runs, interval16{start: uint16(astart), last: uint16(alast)})
 			apos++
 			if apos < alen {
 				astart = a.runs[apos].start
@@ -2608,7 +2570,7 @@ func differenceRunRun(a, b *container) *container {
 		default:
 			// overlap
 			if astart < bstart {
-				output.runs = append(output.runs, interval32{start: uint32(astart), last: uint32(bstart - 1)})
+				output.runs = append(output.runs, interval16{start: uint16(astart), last: uint16(bstart - 1)})
 			}
 			if alast > blast {
 				astart = blast + 1
@@ -2622,7 +2584,7 @@ func differenceRunRun(a, b *container) *container {
 		}
 	}
 	if apos < alen {
-		output.runs = append(output.runs, interval32{start: uint32(astart), last: uint32(alast)})
+		output.runs = append(output.runs, interval16{start: uint16(astart), last: uint16(alast)})
 		apos++
 		if apos < alen {
 			output.runs = append(output.runs, a.runs[apos:]...)
@@ -2874,10 +2836,10 @@ func (op *op) UnmarshalBinary(data []byte) error {
 func (*op) size() int { return 1 + 8 + 4 }
 
 func highbits(v uint64) uint64 { return uint64(v >> 16) }
-func lowbits(v uint64) uint32  { return uint32(v & 0xFFFF) }
+func lowbits(v uint64) uint16  { return uint16(v & 0xFFFF) }
 
 // search32 returns the index of v in a.
-func search32(a []uint32, value uint32) int {
+func search32(a []uint16, value uint16) int {
 	// Optimize for elements and the last element.
 	n := len(a)
 	if n == 0 {
@@ -2986,16 +2948,16 @@ func popcount(x uint64) (n uint64) {
 
 // Returns eof as true if there are no values left in the iterator.
 type containerIterator interface {
-	next() (uint32, bool)
+	next() (uint16, bool)
 }
 
 // arrayIterator represents an iterator over container array values.
 type arrayIterator struct {
-	array []uint32
+	array []uint16
 	i     int
 }
 
-func newArrayIterator(array []uint32) *arrayIterator {
+func newArrayIterator(array []uint16) *arrayIterator {
 	return &arrayIterator{
 		array: array,
 		i:     -1,
@@ -3003,7 +2965,7 @@ func newArrayIterator(array []uint32) *arrayIterator {
 }
 
 // next returns the next value in the array.
-func (itr *arrayIterator) next() (v uint32, eof bool) {
+func (itr *arrayIterator) next() (v uint16, eof bool) {
 
 	itr.i++
 	if itr.i >= len(itr.array) {
@@ -3027,8 +2989,8 @@ func newBitmapIterator(bitmap []uint64) *bitmapIterator {
 
 // next returns the next value in the bitmap.
 // Returns eof as true if there are no values left in the iterator.
-func (itr *bitmapIterator) next() (v uint32, eof bool) {
-	if itr.i+1 >= len(itr.bitmap)*64 {
+func (itr *bitmapIterator) next() (v uint16, eof bool) {
+	if itr.i+1 >= int(len(itr.bitmap)*64) {
 		return 0, true
 	}
 	itr.i++
@@ -3038,14 +3000,14 @@ func (itr *bitmapIterator) next() (v uint32, eof bool) {
 	lb := itr.bitmap[hb] >> (uint(itr.i) % 64)
 	if lb != 0 {
 		itr.i = int(itr.i) + trailingZeroN(lb)
-		return uint32(itr.i), false
+		return uint16(itr.i), false
 	}
 
 	// Otherwise iterate through remaining bitmaps to find next bit.
 	for hb++; hb < len(itr.bitmap); hb++ {
 		if itr.bitmap[hb] != 0 {
 			itr.i = int(hb<<6) + trailingZeroN(itr.bitmap[hb])
-			return uint32(itr.i), false
+			return uint16(itr.i), false
 		}
 	}
 
@@ -3055,7 +3017,7 @@ func (itr *bitmapIterator) next() (v uint32, eof bool) {
 // bufBitmapIterator wraps an iterator to provide the ability to unread values.
 type bufBitmapIterator struct {
 	buf struct {
-		v    uint32
+		v    uint16
 		eof  bool
 		full bool
 	}
@@ -3069,7 +3031,7 @@ func newBufBitmapIterator(itr *bitmapIterator) *bufBitmapIterator {
 
 // next returns the next pair in the bitmap.
 // If a value has been buffered then it is returned and the buffer is cleared.
-func (itr *bufBitmapIterator) next() (v uint32, eof bool) {
+func (itr *bufBitmapIterator) next() (v uint16, eof bool) {
 	if itr.buf.full {
 		itr.buf.full = false
 		return itr.buf.v, itr.buf.eof
@@ -3134,8 +3096,8 @@ func assert(condition bool, format string, a ...interface{}) {
 func xorArrayRun(a, b *container) *container {
 	output := &container{}
 	na, nb := len(a.array), len(b.runs)
-	var vb interval32
-	var va uint32
+	var vb interval16
+	var va uint16
 	last_i, last_j := -1, -1
 	for i, j := 0, 0; i < na || j < nb; {
 		if i < na && i != last_i {
@@ -3148,14 +3110,14 @@ func xorArrayRun(a, b *container) *container {
 		last_j = j
 
 		if i < na && (j >= nb || va < vb.start) { //before
-			output.n += output.runAppendInterval(interval32{start: va, last: va})
+			output.n += output.runAppendInterval(interval16{start: va, last: va})
 			i++
 		} else if j < nb && (i >= na || va > vb.last) { //after
 			output.n += output.runAppendInterval(vb)
 			j++
 		} else if va > vb.start {
 			if va < vb.last {
-				output.n += output.runAppendInterval(interval32{start: vb.start, last: va - 1})
+				output.n += output.runAppendInterval(interval16{start: vb.start, last: va - 1})
 				vb.start = va + 1
 				i++
 				if vb.start > vb.last {
@@ -3187,7 +3149,7 @@ func xorArrayRun(a, b *container) *container {
 }
 
 // xorCompare computes first exclusive run between two runs.
-func xorCompare(x *xorstm) (r1 interval32, has_data bool) {
+func xorCompare(x *xorstm) (r1 interval16, has_data bool) {
 	has_data = false
 	if !x.va_valid || !x.vb_valid {
 		if x.vb_valid {
@@ -3219,7 +3181,7 @@ func xorCompare(x *xorstm) (r1 interval32, has_data bool) {
 	} else if x.va.start <= x.vb.start && x.va.last >= x.vb.last { //vb inside
 		x.vb_valid = false
 		if x.va.start != x.vb.start {
-			r1 = interval32{start: x.va.start, last: x.vb.start - 1}
+			r1 = interval16{start: x.va.start, last: x.vb.start - 1}
 			has_data = true
 		}
 		x.va.start = x.vb.last + 1
@@ -3230,7 +3192,7 @@ func xorCompare(x *xorstm) (r1 interval32, has_data bool) {
 	} else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //va inside
 		x.va_valid = false
 		if x.vb.start != x.va.start {
-			r1 = interval32{start: x.vb.start, last: x.va.start - 1}
+			r1 = interval16{start: x.vb.start, last: x.va.start - 1}
 			has_data = true
 		}
 
@@ -3241,7 +3203,7 @@ func xorCompare(x *xorstm) (r1 interval32, has_data bool) {
 
 	} else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //va first overlap
 		x.va_valid = false
-		r1 = interval32{start: x.va.start, last: x.vb.start - 1}
+		r1 = interval16{start: x.va.start, last: x.vb.start - 1}
 		has_data = true
 		x.vb.start = x.va.last + 1
 		if x.vb.start > x.vb.last {
@@ -3249,7 +3211,7 @@ func xorCompare(x *xorstm) (r1 interval32, has_data bool) {
 		}
 	} else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //vb first overlap
 		x.vb_valid = false
-		r1 = interval32{start: x.vb.start, last: x.va.start - 1}
+		r1 = interval16{start: x.vb.start, last: x.va.start - 1}
 		has_data = true
 		x.va.start = x.vb.last + 1
 		if x.va.start > x.va.last {
@@ -3262,7 +3224,7 @@ func xorCompare(x *xorstm) (r1 interval32, has_data bool) {
 //stm  is state machine used to "xor" iterate over runs.
 type xorstm struct {
 	va_valid, vb_valid bool
-	va, vb             interval32
+	va, vb             interval16
 }
 
 // xorRunRun computes the exclusive or of two run containers.
@@ -3305,7 +3267,7 @@ func xorRunRun(a, b *container) *container {
 
 	}
 
-	if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
+	if output.n < ArrayMaxSize && int(len(output.runs)) > output.n/2 {
 		output.runToArray()
 	} else if len(output.runs) > RunMaxSize {
 		output.runToBitmap()
@@ -3320,7 +3282,7 @@ func xorBitmapRun(a, b *container) *container {
 		output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
 	}
 
-	if output.n < ArrayMaxSize && len(output.runs) > output.n/2 {
+	if output.n < ArrayMaxSize && int(len(output.runs)) > output.n/2 {
 		output.runToArray()
 	} else if len(output.runs) > RunMaxSize {
 		output.runToBitmap()
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 75a686443..f1e64068e 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -22,39 +22,39 @@ import (
 )
 
 // String produces a human viewable string of the contents.
-func (iv interval32) String() string {
+func (iv interval16) String() string {
 	return fmt.Sprintf("[%d, %d]", iv.start, iv.last)
 }
 
 func (c *container) String() string {
-	return fmt.Sprintf("<%s container (%v), n=%d, array[%d], runs[%d], bitmap[%d]>", c.info().Type, &c, c.n, len(c.array), len(c.runs), len(c.bitmap))
+	return fmt.Sprintf("<%s container  n=%d, array[%d], runs[%d], bitmap[%d]> type:%d", c.info().Type, c.n, len(c.array), len(c.runs), len(c.bitmap), c.container_type)
 }
 
 func TestRunAppendInterval(t *testing.T) {
-	a := container{}
+	a := container{container_type: ContainerRun}
 	tests := []struct {
-		base []interval32
-		app  interval32
+		base []interval16
+		app  interval16
 		exp  int
 	}{
 		{
-			base: []interval32{},
-			app:  interval32{start: 22, last: 25},
+			base: []interval16{},
+			app:  interval16{start: 22, last: 25},
 			exp:  4,
 		},
 		{
-			base: []interval32{{start: 20, last: 23}},
-			app:  interval32{start: 22, last: 25},
+			base: []interval16{{start: 20, last: 23}},
+			app:  interval16{start: 22, last: 25},
 			exp:  2,
 		},
 		{
-			base: []interval32{{start: 20, last: 23}},
-			app:  interval32{start: 21, last: 22},
+			base: []interval16{{start: 20, last: 23}},
+			app:  interval16{start: 21, last: 22},
 			exp:  0,
 		},
 		{
-			base: []interval32{{start: 20, last: 23}},
-			app:  interval32{start: 19, last: 25},
+			base: []interval16{{start: 20, last: 23}},
+			app:  interval16{start: 19, last: 25},
 			exp:  2, // runAppendInterval explicitly does not support intervals whose start is < c.runs[-1].start
 		},
 	}
@@ -68,32 +68,32 @@ func TestRunAppendInterval(t *testing.T) {
 
 }
 
-func TestInterval32RunLen(t *testing.T) {
-	iv := interval32{start: 7, last: 9}
+func TestInterval16RunLen(t *testing.T) {
+	iv := interval16{start: 7, last: 9}
 	if iv.runlen() != 3 {
 		t.Fatalf("should be 3")
 	}
-	iv = interval32{start: 7, last: 7}
+	iv = interval16{start: 7, last: 7}
 	if iv.runlen() != 1 {
 		t.Fatalf("should be 1")
 	}
 }
 
 func TestContainerRunAdd(t *testing.T) {
-	c := container{runs: make([]interval32, 0)}
+	c := container{runs: make([]interval16, 0), container_type: ContainerRun}
 	tests := []struct {
-		op  uint32
-		exp []interval32
+		op  uint16
+		exp []interval16
 	}{
-		{1, []interval32{{start: 1, last: 1}}},
-		{2, []interval32{{start: 1, last: 2}}},
-		{4, []interval32{{start: 1, last: 2}, {start: 4, last: 4}}},
-		{3, []interval32{{start: 1, last: 4}}},
-		{10, []interval32{{start: 1, last: 4}, {start: 10, last: 10}}},
-		{7, []interval32{{start: 1, last: 4}, {start: 7, last: 7}, {start: 10, last: 10}}},
-		{6, []interval32{{start: 1, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
-		{0, []interval32{{start: 0, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
-		{8, []interval32{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}},
+		{1, []interval16{{start: 1, last: 1}}},
+		{2, []interval16{{start: 1, last: 2}}},
+		{4, []interval16{{start: 1, last: 2}, {start: 4, last: 4}}},
+		{3, []interval16{{start: 1, last: 4}}},
+		{10, []interval16{{start: 1, last: 4}, {start: 10, last: 10}}},
+		{7, []interval16{{start: 1, last: 4}, {start: 7, last: 7}, {start: 10, last: 10}}},
+		{6, []interval16{{start: 1, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
+		{0, []interval16{{start: 0, last: 4}, {start: 6, last: 7}, {start: 10, last: 10}}},
+		{8, []interval16{{start: 0, last: 4}, {start: 6, last: 8}, {start: 10, last: 10}}},
 	}
 	for _, test := range tests {
 		c.mapped = true
@@ -111,12 +111,12 @@ func TestContainerRunAdd(t *testing.T) {
 }
 
 func TestContainerRunAdd2(t *testing.T) {
-	c := container{runs: make([]interval32, 0)}
+	c := container{runs: make([]interval16, 0), container_type: ContainerRun}
 	ret := c.add(0)
 	if !ret {
 		t.Fatalf("result of adding new bit should be true: %v", c.runs)
 	}
-	if !reflect.DeepEqual(c.runs, []interval32{{start: 0, last: 0}}) {
+	if !reflect.DeepEqual(c.runs, []interval16{{start: 0, last: 0}}) {
 		t.Fatalf("should have 1 run of length 1, but have %v", c.runs)
 	}
 	ret = c.add(0)
@@ -126,7 +126,7 @@ func TestContainerRunAdd2(t *testing.T) {
 }
 
 func TestRunCountRange(t *testing.T) {
-	c := container{runs: make([]interval32, 0)}
+	c := container{runs: make([]interval16, 0), container_type: ContainerRun}
 	cnt := c.runCountRange(2, 9)
 	if cnt != 0 {
 		t.Fatalf("should get 0 from empty container, but got: %v", cnt)
@@ -179,7 +179,7 @@ func TestRunCountRange(t *testing.T) {
 }
 
 func TestRunContains(t *testing.T) {
-	c := container{runs: make([]interval32, 0)}
+	c := container{runs: make([]interval16, 0), container_type: ContainerRun}
 	if c.runContains(5) {
 		t.Fatalf("empty run container should not contain 5")
 	}
@@ -201,10 +201,10 @@ func TestRunContains(t *testing.T) {
 }
 
 func TestBitmapCountRange(t *testing.T) {
-	c := container{}
+	c := container{container_type: ContainerBitmap}
 	tests := []struct {
-		start  uint32
-		end    uint32
+		start  int
+		end    int
 		bitmap []uint64
 		exp    int
 	}{
@@ -228,32 +228,32 @@ func TestBitmapCountRange(t *testing.T) {
 func TestIntersectionCountArrayBitmap2(t *testing.T) {
 	a, b := &container{}, &container{}
 	tests := []struct {
-		array  []uint32
+		array  []uint16
 		bitmap []uint64
 		exp    uint64
 	}{
 		{
-			array:  []uint32{0},
+			array:  []uint16{0},
 			bitmap: []uint64{1},
 			exp:    1,
 		},
 		{
-			array:  []uint32{0, 1},
+			array:  []uint16{0, 1},
 			bitmap: []uint64{3},
 			exp:    2,
 		},
 		{
-			array:  []uint32{64, 128, 129, 2000},
+			array:  []uint16{64, 128, 129, 2000},
 			bitmap: []uint64{932421, 2},
 			exp:    0,
 		},
 		{
-			array:  []uint32{0, 65, 130, 195},
+			array:  []uint16{0, 65, 130, 195},
 			bitmap: []uint64{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
 			exp:    4,
 		},
 		{
-			array:  []uint32{63, 120, 543, 639, 12000},
+			array:  []uint16{63, 120, 543, 639, 12000},
 			bitmap: []uint64{0x8000000000000000, 0, 0, 0, 0, 0, 0, 0, 0, 0x8000000000000000},
 			exp:    2,
 		},
@@ -271,22 +271,22 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
 }
 
 func TestRunRemove(t *testing.T) {
-	c := container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
 	tests := []struct {
-		op     uint32
-		exp    []interval32
+		op     uint16
+		exp    []interval16
 		expRet bool
 	}{
-		{2, []interval32{{start: 3, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
-		{10, []interval32{{start: 3, last: 9}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
-		{12, []interval32{{start: 3, last: 9}, {start: 13, last: 13}, {start: 15, last: 16}}, true},
-		{13, []interval32{{start: 3, last: 9}, {start: 15, last: 16}}, true},
-		{16, []interval32{{start: 3, last: 9}, {start: 15, last: 15}}, true},
-		{6, []interval32{{start: 3, last: 5}, {start: 7, last: 9}, {start: 15, last: 15}}, true},
-		{8, []interval32{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, true},
-		{8, []interval32{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
-		{1, []interval32{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
-		{44, []interval32{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
+		{2, []interval16{{start: 3, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
+		{10, []interval16{{start: 3, last: 9}, {start: 12, last: 13}, {start: 15, last: 16}}, true},
+		{12, []interval16{{start: 3, last: 9}, {start: 13, last: 13}, {start: 15, last: 16}}, true},
+		{13, []interval16{{start: 3, last: 9}, {start: 15, last: 16}}, true},
+		{16, []interval16{{start: 3, last: 9}, {start: 15, last: 15}}, true},
+		{6, []interval16{{start: 3, last: 5}, {start: 7, last: 9}, {start: 15, last: 15}}, true},
+		{8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, true},
+		{8, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
+		{1, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
+		{44, []interval16{{start: 3, last: 5}, {start: 7, last: 7}, {start: 9, last: 9}, {start: 15, last: 15}}, false},
 	}
 
 	for i, test := range tests {
@@ -305,13 +305,13 @@ func TestRunRemove(t *testing.T) {
 }
 
 func TestRunMax(t *testing.T) {
-	c := container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
 	max := c.max()
 	if max != 16 {
 		t.Fatalf("max for %v should be 16", c.runs)
 	}
 
-	c = container{runs: []interval32{}}
+	c = container{runs: []interval16{}}
 	max = c.max()
 	if max != 0 {
 		t.Fatalf("max for %v should be 0", c.runs)
@@ -319,8 +319,8 @@ func TestRunMax(t *testing.T) {
 }
 
 func TestIntersectionCountArrayRun(t *testing.T) {
-	a := &container{array: []uint32{1, 5, 10, 11, 12}}
-	b := &container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	a := &container{array: []uint16{1, 5, 10, 11, 12}}
+	b := &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
 
 	ret := intersectionCountArrayRun(a, b)
 	if ret != 3 {
@@ -330,7 +330,7 @@ func TestIntersectionCountArrayRun(t *testing.T) {
 
 func TestIntersectionCountBitmapRun(t *testing.T) {
 	a := &container{bitmap: []uint64{0x8000000000000000}}
-	b := &container{runs: []interval32{{start: 63, last: 64}}}
+	b := &container{runs: []interval16{{start: 63, last: 64}}}
 
 	ret := intersectionCountBitmapRun(a, b)
 	if ret != 1 {
@@ -338,7 +338,7 @@ func TestIntersectionCountBitmapRun(t *testing.T) {
 	}
 
 	a = &container{bitmap: []uint64{0xF0000001, 0xFF00000000000000, 0xFF000000000000F0, 0x0F0000}}
-	b = &container{runs: []interval32{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
+	b = &container{runs: []interval16{{start: 29, last: 31}, {start: 125, last: 134}, {start: 191, last: 197}, {start: 200, last: 300}}}
 
 	ret = intersectionCountBitmapRun(a, b)
 	if ret != 14 {
@@ -350,40 +350,40 @@ func TestIntersectionCountRunRun(t *testing.T) {
 	a := &container{}
 	b := &container{}
 	tests := []struct {
-		aruns []interval32
-		bruns []interval32
+		aruns []interval16
+		bruns []interval16
 		exp   uint64
 	}{
 		{
-			aruns: []interval32{},
-			bruns: []interval32{{start: 3, last: 8}}, exp: 0},
+			aruns: []interval16{},
+			bruns: []interval16{{start: 3, last: 8}}, exp: 0},
 		{
-			aruns: []interval32{{start: 2, last: 10}},
-			bruns: []interval32{{start: 3, last: 8}}, exp: 6},
+			aruns: []interval16{{start: 2, last: 10}},
+			bruns: []interval16{{start: 3, last: 8}}, exp: 6},
 		{
-			aruns: []interval32{{start: 2, last: 10}},
-			bruns: []interval32{{start: 1, last: 11}}, exp: 9},
+			aruns: []interval16{{start: 2, last: 10}},
+			bruns: []interval16{{start: 1, last: 11}}, exp: 9},
 		{
-			aruns: []interval32{{start: 2, last: 10}},
-			bruns: []interval32{{start: 0, last: 2}}, exp: 1},
+			aruns: []interval16{{start: 2, last: 10}},
+			bruns: []interval16{{start: 0, last: 2}}, exp: 1},
 		{
-			aruns: []interval32{{start: 2, last: 10}},
-			bruns: []interval32{{start: 1, last: 10}}, exp: 9},
+			aruns: []interval16{{start: 2, last: 10}},
+			bruns: []interval16{{start: 1, last: 10}}, exp: 9},
 		{
-			aruns: []interval32{{start: 2, last: 10}},
-			bruns: []interval32{{start: 5, last: 12}}, exp: 6},
+			aruns: []interval16{{start: 2, last: 10}},
+			bruns: []interval16{{start: 5, last: 12}}, exp: 6},
 		{
-			aruns: []interval32{{start: 2, last: 10}},
-			bruns: []interval32{{start: 10, last: 99}}, exp: 1},
+			aruns: []interval16{{start: 2, last: 10}},
+			bruns: []interval16{{start: 10, last: 99}}, exp: 1},
 		{
-			aruns: []interval32{{start: 2, last: 10}, {start: 44, last: 99}},
-			bruns: []interval32{{start: 12, last: 14}}, exp: 0},
+			aruns: []interval16{{start: 2, last: 10}, {start: 44, last: 99}},
+			bruns: []interval16{{start: 12, last: 14}}, exp: 0},
 		{
-			aruns: []interval32{{start: 2, last: 10}, {start: 12, last: 13}},
-			bruns: []interval32{{start: 2, last: 10}, {start: 12, last: 13}}, exp: 11},
+			aruns: []interval16{{start: 2, last: 10}, {start: 12, last: 13}},
+			bruns: []interval16{{start: 2, last: 10}, {start: 12, last: 13}}, exp: 11},
 		{
-			aruns: []interval32{{start: 8, last: 12}, {start: 15, last: 19}},
-			bruns: []interval32{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6},
+			aruns: []interval16{{start: 8, last: 12}, {start: 15, last: 19}},
+			bruns: []interval16{{start: 9, last: 9}, {start: 11, last: 17}}, exp: 6},
 	}
 	for i, test := range tests {
 		a.runs = test.aruns
@@ -399,29 +399,29 @@ func TestIntersectArrayRun(t *testing.T) {
 	a := &container{}
 	b := &container{}
 	tests := []struct {
-		array []uint32
-		runs  []interval32
-		exp   []uint32
+		array []uint16
+		runs  []interval16
+		exp   []uint16
 	}{
 		{
-			array: []uint32{1, 4, 5, 7, 10, 11, 12},
-			runs:  []interval32{{start: 5, last: 10}},
-			exp:   []uint32{5, 7, 10},
+			array: []uint16{1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval16{{start: 5, last: 10}},
+			exp:   []uint16{5, 7, 10},
 		},
 		{
-			array: []uint32{},
-			runs:  []interval32{{start: 5, last: 10}},
-			exp:   []uint32(nil),
+			array: []uint16{},
+			runs:  []interval16{{start: 5, last: 10}},
+			exp:   []uint16(nil),
 		},
 		{
-			array: []uint32{1, 4, 5, 7, 10, 11, 12},
-			runs:  []interval32{},
-			exp:   []uint32(nil),
+			array: []uint16{1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval16{},
+			exp:   []uint16(nil),
 		},
 		{
-			array: []uint32{0, 1, 4, 5, 7, 10, 11, 12},
-			runs:  []interval32{{start: 0, last: 5}, {start: 7, last: 7}},
-			exp:   []uint32{0, 1, 4, 5, 7},
+			array: []uint16{0, 1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval16{{start: 0, last: 5}, {start: 7, last: 7}},
+			exp:   []uint16{0, 1, 4, 5, 7},
 		},
 	}
 
@@ -439,45 +439,45 @@ func TestIntersectRunRun(t *testing.T) {
 	a := &container{}
 	b := &container{}
 	tests := []struct {
-		aruns []interval32
-		bruns []interval32
-		exp   []interval32
+		aruns []interval16
+		bruns []interval16
+		exp   []interval16
 		expN  int
 	}{
 		{
-			aruns: []interval32{},
-			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32(nil),
+			aruns: []interval16{},
+			bruns: []interval16{{start: 5, last: 10}},
+			exp:   []interval16(nil),
 			expN:  0,
 		},
 		{
-			aruns: []interval32{{start: 5, last: 12}},
-			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 5, last: 10}},
+			aruns: []interval16{{start: 5, last: 12}},
+			bruns: []interval16{{start: 5, last: 10}},
+			exp:   []interval16{{start: 5, last: 10}},
 			expN:  6,
 		},
 		{
-			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
-			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 5, last: 5}, {start: 7, last: 10}},
+			aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			bruns: []interval16{{start: 5, last: 10}},
+			exp:   []interval16{{start: 5, last: 5}, {start: 7, last: 10}},
 			expN:  5,
 		},
 		{
-			aruns: []interval32{{start: 20, last: 30}},
-			bruns: []interval32{{start: 5, last: 10}, {start: 19, last: 21}},
-			exp:   []interval32{{start: 20, last: 21}},
+			aruns: []interval16{{start: 20, last: 30}},
+			bruns: []interval16{{start: 5, last: 10}, {start: 19, last: 21}},
+			exp:   []interval16{{start: 20, last: 21}},
 			expN:  2,
 		},
 		{
-			aruns: []interval32{{start: 5, last: 10}},
-			bruns: []interval32{{start: 7, last: 12}},
-			exp:   []interval32{{start: 7, last: 10}},
+			aruns: []interval16{{start: 5, last: 10}},
+			bruns: []interval16{{start: 7, last: 12}},
+			exp:   []interval16{{start: 7, last: 10}},
 			expN:  4,
 		},
 		{
-			aruns: []interval32{{start: 5, last: 12}},
-			bruns: []interval32{{start: 7, last: 10}},
-			exp:   []interval32{{start: 7, last: 10}},
+			aruns: []interval16{{start: 5, last: 12}},
+			bruns: []interval16{{start: 7, last: 10}},
+			exp:   []interval16{{start: 7, last: 10}},
 			expN:  4,
 		},
 	}
@@ -500,37 +500,37 @@ func TestIntersectBitmapRunBitmap(t *testing.T) {
 	b := &container{}
 	tests := []struct {
 		bitmap []uint64
-		runs   []interval32
+		runs   []interval16
 		exp    []uint64
 		expN   int
 	}{
 		{
 			bitmap: []uint64{1},
-			runs:   []interval32{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}},
+			runs:   []interval16{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}},
 			exp:    []uint64{1},
 			expN:   1,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
-			runs:   []interval32{{start: 1, last: 1}},
+			runs:   []interval16{{start: 1, last: 1}},
 			exp:    []uint64{2},
 			expN:   1,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
-			runs:   []interval32{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}},
+			runs:   []interval16{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}},
 			exp:    []uint64{0xe000000000001C02},
 			expN:   7,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF},
-			runs:   []interval32{{start: 1, last: 1}, {start: 61, last: 77}},
+			runs:   []interval16{{start: 1, last: 1}, {start: 61, last: 77}},
 			exp:    []uint64{0xE000000000000002, 0x00000000000003FFF},
 			expN:   18,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1},
-			runs:   []interval32{{start: 63, last: 10000}},
+			runs:   []interval16{{start: 63, last: 10000}},
 			exp:    []uint64{0x8000000000000000, 1, 1, 1, 0xA, 1, 1, 0, 1},
 			expN:   9,
 		},
@@ -564,38 +564,38 @@ func TestIntersectBitmapRunArray(t *testing.T) {
 	b := &container{}
 	tests := []struct {
 		bitmap []uint64
-		runs   []interval32
-		exp    []uint32
+		runs   []interval16
+		exp    []uint16
 		expN   int
 	}{
 		{
 			bitmap: []uint64{1},
-			runs:   []interval32{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}},
-			exp:    []uint32{0},
+			runs:   []interval16{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 4096}},
+			exp:    []uint16{0},
 			expN:   1,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
-			runs:   []interval32{{start: 1, last: 1}},
-			exp:    []uint32{1},
+			runs:   []interval16{{start: 1, last: 1}},
+			exp:    []uint16{1},
 			expN:   1,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
-			runs:   []interval32{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}},
-			exp:    []uint32{1, 10, 11, 12, 61, 62, 63},
+			runs:   []interval16{{start: 1, last: 1}, {start: 10, last: 12}, {start: 61, last: 77}},
+			exp:    []uint16{1, 10, 11, 12, 61, 62, 63},
 			expN:   7,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF},
-			runs:   []interval32{{start: 1, last: 1}, {start: 61, last: 68}},
-			exp:    []uint32{1, 61, 62, 63, 64, 65, 66, 67, 68},
+			runs:   []interval16{{start: 1, last: 1}, {start: 61, last: 68}},
+			exp:    []uint16{1, 61, 62, 63, 64, 65, 66, 67, 68},
 			expN:   9,
 		},
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 1, 1, 1, 0xA, 1, 1, 0, 1},
-			runs:   []interval32{{start: 63, last: 10000}},
-			exp:    []uint32{63, 64, 128, 192, 257, 259, 320, 384, 512},
+			runs:   []interval16{{start: 63, last: 10000}},
+			exp:    []uint16{63, 64, 128, 192, 257, 259, 320, 384, 512},
 			expN:   9,
 		},
 	}
@@ -620,22 +620,22 @@ func TestUnionMixed(t *testing.T) {
 	b := &container{}
 	c := &container{}
 
-	a.runs = []interval32{{start: 5, last: 10}}
+	a.runs = []interval16{{start: 5, last: 10}}
 	a.n = 6
-	b.array = []uint32{1, 4, 5, 7, 10, 11, 12}
+	b.array = []uint16{1, 4, 5, 7, 10, 11, 12}
 	b.n = 7
 	res := union(a, b)
-	if !reflect.DeepEqual(res.array, []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}) {
-		t.Fatalf("test #1 expected %v, but got %v", []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, res.array)
 	}
 	res = union(b, a)
-	if !reflect.DeepEqual(res.array, []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}) {
-		t.Fatalf("test #2 expected %v, but got %v", []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}) {
+		t.Fatalf("test #2 expected %v, but got %v", []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}, res.array)
 	}
 
 	res = union(a, a)
-	if !reflect.DeepEqual(res.runs, []interval32{{start: 5, last: 10}}) {
-		t.Fatalf("test #3 expected %v, but got %v", []interval32{{start: 5, last: 10}}, res.runs)
+	if !reflect.DeepEqual(res.runs, []interval16{{start: 5, last: 10}}) {
+		t.Fatalf("test #3 expected %v, but got %v", []interval16{{start: 5, last: 10}}, res.runs)
 	}
 	c.bitmap = []uint64{0x3}
 	c.n = 2
@@ -648,12 +648,12 @@ func TestUnionMixed(t *testing.T) {
 		t.Fatalf("test #5 expected %v, but got %v", []uint64{2019}, res.bitmap)
 	}
 	res = union(b, c)
-	if !reflect.DeepEqual(res.array, []uint32{0, 1, 4, 5, 7, 10, 11, 12}) {
-		t.Fatalf("test #6 expected %v, but got %v", []uint32{0, 1, 4, 5, 7, 10, 11, 12}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{0, 1, 4, 5, 7, 10, 11, 12}) {
+		t.Fatalf("test #6 expected %v, but got %v", []uint16{0, 1, 4, 5, 7, 10, 11, 12}, res.array)
 	}
 	res = union(c, b)
-	if !reflect.DeepEqual(res.array, []uint32{0, 1, 4, 5, 7, 10, 11, 12}) {
-		t.Fatalf("test #6 expected %v, but got %v", []uint32{0, 1, 4, 5, 7, 10, 11, 12}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{0, 1, 4, 5, 7, 10, 11, 12}) {
+		t.Fatalf("test #6 expected %v, but got %v", []uint16{0, 1, 4, 5, 7, 10, 11, 12}, res.array)
 	}
 
 }
@@ -662,42 +662,42 @@ func TestIntersectMixed(t *testing.T) {
 	b := &container{}
 	c := &container{}
 
-	a.runs = []interval32{{start: 5, last: 10}}
+	a.runs = []interval16{{start: 5, last: 10}}
 	a.n = 6
-	b.array = []uint32{1, 4, 5, 7, 10, 11, 12}
+	b.array = []uint16{1, 4, 5, 7, 10, 11, 12}
 	b.n = 7
 	res := intersect(a, b)
-	if !reflect.DeepEqual(res.array, []uint32{5, 7, 10}) {
-		t.Fatalf("test #1 expected %v, but got %v", []uint32{5, 7, 10}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array)
 	}
 	res = intersect(b, a)
-	if !reflect.DeepEqual(res.array, []uint32{5, 7, 10}) {
-		t.Fatalf("test #1 expected %v, but got %v", []uint32{5, 7, 10}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array)
 	}
 
 	res = intersect(a, a)
-	if !reflect.DeepEqual(res.runs, []interval32{{start: 5, last: 10}}) {
-		t.Fatalf("test #3 expected %v, but got %v", []interval32{{start: 5, last: 10}}, res.runs)
+	if !reflect.DeepEqual(res.runs, []interval16{{start: 5, last: 10}}) {
+		t.Fatalf("test #3 expected %v, but got %v", []interval16{{start: 5, last: 10}}, res.runs)
 	}
 	c.bitmap = []uint64{0x60}
 	c.n = 2
 	res = intersect(c, a)
-	if !reflect.DeepEqual(res.array, []uint32{5, 6}) {
-		t.Fatalf("test #4 expected %v, but got %v", []uint32{6}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{5, 6}) {
+		t.Fatalf("test #4 expected %v, but got %v", []uint16{6}, res.array)
 	}
 
 	res = intersect(a, c)
-	if !reflect.DeepEqual(res.array, []uint32{5, 6}) {
-		t.Fatalf("test #5 expected %v, but got %v", []uint32{6}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{5, 6}) {
+		t.Fatalf("test #5 expected %v, but got %v", []uint16{6}, res.array)
 	}
 
 	res = intersect(b, c)
-	if !reflect.DeepEqual(res.array, []uint32{5}) {
-		t.Fatalf("test #6 expected %v, but got %v", []uint32{5}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{5}) {
+		t.Fatalf("test #6 expected %v, but got %v", []uint16{5}, res.array)
 	}
 	res = intersect(c, b)
-	if !reflect.DeepEqual(res.array, []uint32{5}) {
-		t.Fatalf("test #7 expected %v, but got %v", []uint32{5}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{5}) {
+		t.Fatalf("test #7 expected %v, but got %v", []uint16{5}, res.array)
 	}
 
 }
@@ -707,28 +707,28 @@ func TestDifferenceMixed(t *testing.T) {
 	c := &container{}
 	d := &container{}
 
-	a.runs = []interval32{{start: 5, last: 10}}
+	a.runs = []interval16{{start: 5, last: 10}}
 	a.n = a.runCountRange(0, 100)
 
-	b.array = []uint32{0, 2, 4, 6, 8, 10, 12}
+	b.array = []uint16{0, 2, 4, 6, 8, 10, 12}
 	b.n = len(b.array)
 
-	d.array = []uint32{1, 3, 5, 7, 9, 11, 12}
+	d.array = []uint16{1, 3, 5, 7, 9, 11, 12}
 	d.n = len(d.array)
 
 	res := difference(a, b)
 
-	if !reflect.DeepEqual(res.array, []uint32{5, 7, 9}) {
-		t.Fatalf("test #1 expected %v, but got %v", []uint32{5, 7, 9}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{5, 7, 9}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 9}, res.array)
 	}
 
 	res = difference(b, a)
-	if !reflect.DeepEqual(res.array, []uint32{0, 2, 4, 12}) {
-		t.Fatalf("test #2 expected %v, but got %v", []uint32{0, 2, 4, 12}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{0, 2, 4, 12}) {
+		t.Fatalf("test #2 expected %v, but got %v", []uint16{0, 2, 4, 12}, res.array)
 	}
 
 	res = difference(a, a)
-	if !reflect.DeepEqual(res.runs, []interval32{}) {
+	if !reflect.DeepEqual(res.runs, []interval16{}) {
 		t.Fatalf("test #3 expected empty but got %v", res.runs)
 	}
 
@@ -736,22 +736,22 @@ func TestDifferenceMixed(t *testing.T) {
 	c.n = c.countRange(0, 100)
 	res = difference(c, a)
 	if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) {
-		t.Fatalf("test #4 expected %v, but got %v", []uint32{4}, res.bitmap)
+		t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap)
 	}
 
 	res = difference(a, c)
-	if !reflect.DeepEqual(res.runs, []interval32{{start: 7, last: 10}}) {
-		t.Fatalf("test #5 expected %v, but got %v", []interval32{{start: 7, last: 10}}, res.runs)
+	if !reflect.DeepEqual(res.runs, []interval16{{start: 7, last: 10}}) {
+		t.Fatalf("test #5 expected %v, but got %v", []interval16{{start: 7, last: 10}}, res.runs)
 	}
 
 	res = difference(b, c)
-	if !reflect.DeepEqual(res.array, []uint32{0, 4, 8, 10, 12}) {
-		t.Fatalf("test #6 expected %v, but got %v", []uint32{0, 4, 8, 10, 12}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{0, 4, 8, 10, 12}) {
+		t.Fatalf("test #6 expected %v, but got %v", []uint16{0, 4, 8, 10, 12}, res.array)
 	}
 
 	res = difference(c, b)
-	if !reflect.DeepEqual(res.array, []uint32{5}) {
-		t.Fatalf("test #7 expected %v, but got %v", []uint32{5}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{5}) {
+		t.Fatalf("test #7 expected %v, but got %v", []uint16{5}, res.array)
 	}
 
 	res = difference(b, b)
@@ -765,13 +765,13 @@ func TestDifferenceMixed(t *testing.T) {
 	}
 
 	res = difference(d, b)
-	if !reflect.DeepEqual(res.array, []uint32{1, 3, 5, 7, 9, 11}) {
-		t.Fatalf("test #10 expected %v, but got %d", []uint32{1, 3, 5, 7, 9, 11}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{1, 3, 5, 7, 9, 11}) {
+		t.Fatalf("test #10 expected %v, but got %d", []uint16{1, 3, 5, 7, 9, 11}, res.array)
 	}
 
 	res = difference(b, d)
-	if !reflect.DeepEqual(res.array, []uint32{0, 2, 4, 6, 8, 10}) {
-		t.Fatalf("test #11 expected %v, but got %d", []uint32{0, 2, 4, 6, 8, 10}, res.array)
+	if !reflect.DeepEqual(res.array, []uint16{0, 2, 4, 6, 8, 10}) {
+		t.Fatalf("test #11 expected %v, but got %d", []uint16{0, 2, 4, 6, 8, 10}, res.array)
 	}
 
 }
@@ -780,49 +780,49 @@ func TestUnionRunRun(t *testing.T) {
 	a := &container{}
 	b := &container{}
 	tests := []struct {
-		aruns []interval32
-		bruns []interval32
-		exp   []interval32
+		aruns []interval16
+		bruns []interval16
+		exp   []interval16
 	}{
 		{
-			aruns: []interval32{},
-			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 5, last: 10}},
+			aruns: []interval16{},
+			bruns: []interval16{{start: 5, last: 10}},
+			exp:   []interval16{{start: 5, last: 10}},
 		},
 		{
-			aruns: []interval32{{start: 5, last: 12}},
-			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 5, last: 12}},
+			aruns: []interval16{{start: 5, last: 12}},
+			bruns: []interval16{{start: 5, last: 10}},
+			exp:   []interval16{{start: 5, last: 12}},
 		},
 		{
-			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
-			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 1, last: 3}, {start: 5, last: 12}},
+			aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			bruns: []interval16{{start: 5, last: 10}},
+			exp:   []interval16{{start: 1, last: 3}, {start: 5, last: 12}},
 		},
 		{
-			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
-			bruns: []interval32{{start: 2, last: 65535}},
-			exp:   []interval32{{start: 1, last: 65535}},
+			aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			bruns: []interval16{{start: 2, last: 65535}},
+			exp:   []interval16{{start: 1, last: 65535}},
 		},
 		{
-			aruns: []interval32{{start: 2, last: 65535}},
-			bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
-			exp:   []interval32{{start: 1, last: 65535}},
+			aruns: []interval16{{start: 2, last: 65535}},
+			bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			exp:   []interval16{{start: 1, last: 65535}},
 		},
 		{
-			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
-			bruns: []interval32{{start: 0, last: 65535}},
-			exp:   []interval32{{start: 0, last: 65535}},
+			aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			bruns: []interval16{{start: 0, last: 65535}},
+			exp:   []interval16{{start: 0, last: 65535}},
 		},
 		{
-			aruns: []interval32{{start: 0, last: 65535}},
-			bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
-			exp:   []interval32{{start: 0, last: 65535}},
+			aruns: []interval16{{start: 0, last: 65535}},
+			bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 8}, {start: 9, last: 12}},
+			exp:   []interval16{{start: 0, last: 65535}},
 		},
 		{
-			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}},
-			bruns: []interval32{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}},
-			exp:   []interval32{{start: 1, last: 9}, {start: 12, last: 27}, {start: 33, last: 34}},
+			aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}},
+			bruns: []interval16{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}},
+			exp:   []interval16{{start: 1, last: 9}, {start: 12, last: 27}, {start: 33, last: 34}},
 		},
 	}
 	for i, test := range tests {
@@ -839,29 +839,29 @@ func TestUnionArrayRun(t *testing.T) {
 	a := &container{}
 	b := &container{}
 	tests := []struct {
-		array []uint32
-		runs  []interval32
-		exp   []uint32
+		array []uint16
+		runs  []interval16
+		exp   []uint16
 	}{
 		{
-			array: []uint32{1, 4, 5, 7, 10, 11, 12},
-			runs:  []interval32{{start: 5, last: 10}},
-			exp:   []uint32{1, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+			array: []uint16{1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval16{{start: 5, last: 10}},
+			exp:   []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12},
 		},
 		{
-			array: []uint32{},
-			runs:  []interval32{{start: 5, last: 10}},
-			exp:   []uint32{5, 6, 7, 8, 9, 10},
+			array: []uint16{},
+			runs:  []interval16{{start: 5, last: 10}},
+			exp:   []uint16{5, 6, 7, 8, 9, 10},
 		},
 		{
-			array: []uint32{1, 4, 5, 7, 10, 11, 12},
-			runs:  []interval32{},
-			exp:   []uint32{1, 4, 5, 7, 10, 11, 12},
+			array: []uint16{1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval16{},
+			exp:   []uint16{1, 4, 5, 7, 10, 11, 12},
 		},
 		{
-			array: []uint32{0, 1, 4, 5, 7, 10, 11, 12},
-			runs:  []interval32{{start: 0, last: 5}, {start: 7, last: 7}},
-			exp:   []uint32{0, 1, 2, 3, 4, 5, 7, 10, 11, 12},
+			array: []uint16{0, 1, 4, 5, 7, 10, 11, 12},
+			runs:  []interval16{{start: 0, last: 5}, {start: 7, last: 7}},
+			exp:   []uint16{0, 1, 2, 3, 4, 5, 7, 10, 11, 12},
 		},
 	}
 
@@ -918,15 +918,15 @@ func TestBitmapSetRange(t *testing.T) {
 func TestArrayToBitmap(t *testing.T) {
 	a := &container{}
 	tests := []struct {
-		array []uint32
+		array []uint16
 		exp   []uint64
 	}{
 		{
-			array: []uint32{},
+			array: []uint16{},
 			exp:   []uint64{},
 		},
 		{
-			array: []uint32{0, 1, 2, 3},
+			array: []uint16{0, 1, 2, 3},
 			exp:   []uint64{0xF},
 		},
 	}
@@ -950,15 +950,15 @@ func TestBitmapToArray(t *testing.T) {
 	a := &container{}
 	tests := []struct {
 		bitmap []uint64
-		exp    []uint32
+		exp    []uint16
 	}{
 		{
 			bitmap: []uint64{},
-			exp:    []uint32{},
+			exp:    []uint16{},
 		},
 		{
 			bitmap: []uint64{0xF},
-			exp:    []uint32{0, 1, 2, 3},
+			exp:    []uint16{0, 1, 2, 3},
 		},
 	}
 	for i, test := range tests {
@@ -980,27 +980,27 @@ func TestBitmapToArray(t *testing.T) {
 func TestRunToBitmap(t *testing.T) {
 	a := &container{}
 	tests := []struct {
-		runs []interval32
+		runs []interval16
 		exp  []uint64
 	}{
 		{
-			runs: []interval32{},
+			runs: []interval16{},
 			exp:  []uint64{},
 		},
 		{
-			runs: []interval32{{start: 0, last: 0}},
+			runs: []interval16{{start: 0, last: 0}},
 			exp:  []uint64{1},
 		},
 		{
-			runs: []interval32{{start: 0, last: 4}},
+			runs: []interval16{{start: 0, last: 4}},
 			exp:  []uint64{31},
 		},
 		{
-			runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
+			runs: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
 			exp:  []uint64{155876},
 		},
 		{
-			runs: []interval32{{start: 0, last: 3}, {start: 60, last: 67}},
+			runs: []interval16{{start: 0, last: 3}, {start: 60, last: 67}},
 			exp:  []uint64{0xF00000000000000F, 0x000000000000000F},
 		},
 	}
@@ -1026,51 +1026,51 @@ func TestBitmapToRun(t *testing.T) {
 	a := &container{}
 	tests := []struct {
 		bitmap []uint64
-		exp    []interval32
+		exp    []interval16
 	}{
 		{
 			// empty run
 			bitmap: []uint64{},
-			exp:    []interval32{},
+			exp:    []interval16{},
 		},
 		{
 			// single-bit run
 			bitmap: []uint64{1},
-			exp:    []interval32{{start: 0, last: 0}},
+			exp:    []interval16{{start: 0, last: 0}},
 		},
 		{
 			// single multi-bit run in one word
 			bitmap: []uint64{31},
-			exp:    []interval32{{start: 0, last: 4}},
+			exp:    []interval16{{start: 0, last: 4}},
 		},
 		{
 			// multiple runs in one word
 			bitmap: []uint64{155876},
-			exp:    []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
+			exp:    []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
 		},
 		{
 			// span two words, both mixed
 			bitmap: []uint64{0xF00000000000000F, 0x000000000000000F},
-			exp:    []interval32{{start: 0, last: 3}, {start: 60, last: 67}},
+			exp:    []interval16{{start: 0, last: 3}, {start: 60, last: 67}},
 		},
 		{
 			// span two words, first = maxBitmap
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF, 0xF},
-			exp:    []interval32{{start: 0, last: 67}},
+			exp:    []interval16{{start: 0, last: 67}},
 		},
 		{
 			// span two words, second = maxBitmap
 			bitmap: []uint64{0xF000000000000000, 0xFFFFFFFFFFFFFFFF},
-			exp:    []interval32{{start: 60, last: 127}},
+			exp:    []interval16{{start: 60, last: 127}},
 		},
 		{
 			// span three words
 			bitmap: []uint64{0xF000000000000000, 0xFFFFFFFFFFFFFFFF, 0xF},
-			exp:    []interval32{{start: 60, last: 131}},
+			exp:    []interval16{{start: 60, last: 131}},
 		},
 		{
 			bitmap: make([]uint64, bitmapN),
-			exp:    []interval32{{start: 65408, last: 65535}},
+			exp:    []interval16{{start: 65408, last: 65535}},
 		},
 	}
 	tests[8].bitmap[1022] = 0xFFFFFFFFFFFFFFFF
@@ -1094,30 +1094,30 @@ func TestBitmapToRun(t *testing.T) {
 func TestArrayToRun(t *testing.T) {
 	a := &container{}
 	tests := []struct {
-		array []uint32
-		exp   []interval32
+		array []uint16
+		exp   []interval16
 	}{
 		{
-			array: []uint32{},
-			exp:   []interval32{},
+			array: []uint16{},
+			exp:   []interval16{},
 		},
 		{
-			array: []uint32{0},
-			exp:   []interval32{{start: 0, last: 0}},
+			array: []uint16{0},
+			exp:   []interval16{{start: 0, last: 0}},
 		},
 		{
-			array: []uint32{0, 1, 2, 3, 4},
-			exp:   []interval32{{start: 0, last: 4}},
+			array: []uint16{0, 1, 2, 3, 4},
+			exp:   []interval16{{start: 0, last: 4}},
 		},
 		{
-			array: []uint32{2, 5, 6, 7, 13, 14, 17},
-			exp:   []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
+			array: []uint16{2, 5, 6, 7, 13, 14, 17},
+			exp:   []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
 		},
 	}
 
 	for i, test := range tests {
 		a.array = test.array
-		a.n = len(test.array)
+		a.n = int(len(test.array))
 		a.arrayToRun()
 		if !reflect.DeepEqual(a.runs, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs)
@@ -1128,24 +1128,24 @@ func TestArrayToRun(t *testing.T) {
 func TestRunToArray(t *testing.T) {
 	a := &container{}
 	tests := []struct {
-		runs []interval32
-		exp  []uint32
+		runs []interval16
+		exp  []uint16
 	}{
 		{
-			runs: []interval32{},
-			exp:  []uint32{},
+			runs: []interval16{},
+			exp:  []uint16{},
 		},
 		{
-			runs: []interval32{{start: 0, last: 0}},
-			exp:  []uint32{0},
+			runs: []interval16{{start: 0, last: 0}},
+			exp:  []uint16{0},
 		},
 		{
-			runs: []interval32{{start: 0, last: 4}},
-			exp:  []uint32{0, 1, 2, 3, 4},
+			runs: []interval16{{start: 0, last: 4}},
+			exp:  []uint16{0, 1, 2, 3, 4},
 		},
 		{
-			runs: []interval32{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
-			exp:  []uint32{2, 5, 6, 7, 13, 14, 17},
+			runs: []interval16{{start: 2, last: 2}, {start: 5, last: 7}, {start: 13, last: 14}, {start: 17, last: 17}},
+			exp:  []uint16{2, 5, 6, 7, 13, 14, 17},
 		},
 	}
 
@@ -1208,13 +1208,13 @@ func TestUnionBitmapRun(t *testing.T) {
 	b := &container{}
 	tests := []struct {
 		bitmap []uint64
-		runs   []interval32
+		runs   []interval16
 		exp    []uint64
 		expN   int
 	}{
 		{
 			bitmap: []uint64{2},
-			runs:   []interval32{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 78}},
+			runs:   []interval16{{start: 0, last: 0}, {start: 2, last: 5}, {start: 62, last: 71}, {start: 77, last: 78}},
 			exp:    []uint64{0xC00000000000003F, 0x60FF},
 			expN:   18,
 		},
@@ -1294,31 +1294,31 @@ func TestBitmapCountRuns(t *testing.T) {
 func TestArrayCountRuns(t *testing.T) {
 	c := &container{}
 	tests := []struct {
-		array []uint32
+		array []uint16
 		exp   int
 	}{
 		{
-			array: []uint32{},
+			array: []uint16{},
 			exp:   0,
 		},
 		{
-			array: []uint32{0},
+			array: []uint16{0},
 			exp:   1,
 		},
 		{
-			array: []uint32{1},
+			array: []uint16{1},
 			exp:   1,
 		},
 		{
-			array: []uint32{1, 2, 3, 5},
+			array: []uint16{1, 2, 3, 5},
 			exp:   2,
 		},
 		{
-			array: []uint32{0, 1, 3, 9, 2048, 4096, 4097, 65534, 65535},
+			array: []uint16{0, 1, 3, 9, 2048, 4096, 4097, 65534, 65535},
 			exp:   6,
 		},
 		{
-			array: []uint32{0, 10, 11, 12},
+			array: []uint16{0, 10, 11, 12},
 			exp:   2,
 		},
 	}
@@ -1336,14 +1336,14 @@ func TestDifferenceArrayRun(t *testing.T) {
 	a := &container{}
 	b := &container{}
 	tests := []struct {
-		array []uint32
-		runs  []interval32
-		exp   []uint32
+		array []uint16
+		runs  []interval16
+		exp   []uint16
 	}{
 		{
-			array: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
-			runs:  []interval32{{start: 5, last: 10}},
-			exp:   []uint32{0, 1, 2, 3, 4, 11, 12},
+			array: []uint16{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+			runs:  []interval16{{start: 5, last: 10}},
+			exp:   []uint16{0, 1, 2, 3, 4, 11, 12},
 		},
 	}
 	for i, test := range tests {
@@ -1362,29 +1362,29 @@ func TestDifferenceRunArray(t *testing.T) {
 	a := &container{}
 	b := &container{}
 	tests := []struct {
-		runs  []interval32
-		array []uint32
-		exp   []interval32
+		runs  []interval16
+		array []uint16
+		exp   []interval16
 	}{
 		{
-			runs:  []interval32{{start: 0, last: 12}},
-			array: []uint32{5, 6, 7, 8, 9, 10},
-			exp:   []interval32{{start: 0, last: 4}, {start: 11, last: 12}},
+			runs:  []interval16{{start: 0, last: 12}},
+			array: []uint16{5, 6, 7, 8, 9, 10},
+			exp:   []interval16{{start: 0, last: 4}, {start: 11, last: 12}},
 		},
 		{
-			runs:  []interval32{{start: 0, last: 12}},
-			array: []uint32{0, 1, 2, 3},
-			exp:   []interval32{{start: 4, last: 12}},
+			runs:  []interval16{{start: 0, last: 12}},
+			array: []uint16{0, 1, 2, 3},
+			exp:   []interval16{{start: 4, last: 12}},
 		},
 		{
-			runs:  []interval32{{start: 0, last: 12}},
-			array: []uint32{9, 10, 11, 12, 13},
-			exp:   []interval32{{start: 0, last: 8}},
+			runs:  []interval16{{start: 0, last: 12}},
+			array: []uint16{9, 10, 11, 12, 13},
+			exp:   []interval16{{start: 0, last: 8}},
 		},
 		{
-			runs:  []interval32{{start: 1, last: 12}},
-			array: []uint32{0, 9, 10, 11, 12, 13},
-			exp:   []interval32{{start: 1, last: 8}},
+			runs:  []interval16{{start: 1, last: 12}},
+			array: []uint16{0, 9, 10, 11, 12, 13},
+			exp:   []interval16{{start: 1, last: 8}},
 		},
 	}
 	for i, test := range tests {
@@ -1403,39 +1403,39 @@ func TestDifferenceRunBitmap(t *testing.T) {
 	a := &container{}
 	b := &container{bitmap: make([]uint64, bitmapN)}
 	tests := []struct {
-		runs   []interval32
+		runs   []interval16
 		bitmap []uint64
-		exp    []interval32
+		exp    []interval16
 	}{
 		{
-			runs:   []interval32{{start: 0, last: 63}},
+			runs:   []interval16{{start: 0, last: 63}},
 			bitmap: []uint64{0x0000FFFF000000F0},
-			exp:    []interval32{{start: 0, last: 3}, {start: 8, last: 31}, {start: 48, last: 63}},
+			exp:    []interval16{{start: 0, last: 3}, {start: 8, last: 31}, {start: 48, last: 63}},
 		},
 		{
-			runs:   []interval32{{start: 0, last: 63}},
+			runs:   []interval16{{start: 0, last: 63}},
 			bitmap: []uint64{0x8000000000000000},
-			exp:    []interval32{{start: 0, last: 62}},
+			exp:    []interval16{{start: 0, last: 62}},
 		},
 		{
-			runs:   []interval32{{start: 0, last: 63}},
+			runs:   []interval16{{start: 0, last: 63}},
 			bitmap: []uint64{0x0000000000000001},
-			exp:    []interval32{{start: 1, last: 63}},
+			exp:    []interval16{{start: 1, last: 63}},
 		},
 		{
-			runs:   []interval32{{start: 0, last: 63}},
+			runs:   []interval16{{start: 0, last: 63}},
 			bitmap: []uint64{0x0, 0x0000000000000001},
-			exp:    []interval32{{start: 0, last: 63}},
+			exp:    []interval16{{start: 0, last: 63}},
 		},
 		{
-			runs:   []interval32{{start: 0, last: 65}},
+			runs:   []interval16{{start: 0, last: 65}},
 			bitmap: []uint64{0x0, 0x0000000000000001},
-			exp:    []interval32{{start: 0, last: 63}, {start: 65, last: 65}},
+			exp:    []interval16{{start: 0, last: 63}, {start: 65, last: 65}},
 		},
 		{
-			runs:   []interval32{{start: 0, last: 65}},
+			runs:   []interval16{{start: 0, last: 65}},
 			bitmap: []uint64{0x0, 0x8000000000000000},
-			exp:    []interval32{{start: 0, last: 65}},
+			exp:    []interval16{{start: 0, last: 65}},
 		},
 	}
 	for i, test := range tests {
@@ -1457,12 +1457,12 @@ func TestDifferenceBitmapRun(t *testing.T) {
 	b := &container{}
 	tests := []struct {
 		bitmap []uint64
-		runs   []interval32
+		runs   []interval16
 		exp    []uint64
 	}{
 		{
 			bitmap: []uint64{0xFFFFFFFFFFFFFFFF},
-			runs:   []interval32{{start: 4, last: 7}, {start: 32, last: 47}},
+			runs:   []interval16{{start: 4, last: 7}, {start: 32, last: 47}},
 			exp:    []uint64{0xFFFF0000FFFFFF0F},
 		},
 	}
@@ -1485,13 +1485,13 @@ func TestDifferenceBitmapArray(t *testing.T) {
 	b := &container{}
 	tests := []struct {
 		bitmap []uint64
-		array  []uint32
-		exp    []uint32
+		array  []uint16
+		exp    []uint16
 	}{
 		{
 			bitmap: []uint64{0xFF0F},
-			array:  []uint32{0, 1, 2, 3, 4, 5, 6, 7, 10},
-			exp:    []uint32{8, 9, 11, 12, 13, 14, 15},
+			array:  []uint16{0, 1, 2, 3, 4, 5, 6, 7, 10},
+			exp:    []uint16{8, 9, 11, 12, 13, 14, 15},
 		},
 	}
 	for i, test := range tests {
@@ -1509,17 +1509,17 @@ func TestDifferenceBitmapBitmap(t *testing.T) {
 	tests := []struct {
 		abitmap []uint64
 		bbitmap []uint64
-		exp     []uint32
+		exp     []uint16
 	}{
 		{
 			abitmap: []uint64{0xFF00FFFFFFFFFFFF},
 			bbitmap: []uint64{0xFFFFFFFFFFFFF000},
-			exp:     []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
+			exp:     []uint16{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11},
 		},
 		{
 			abitmap: []uint64{0xF},
 			bbitmap: []uint64{},
-			exp:     []uint32{0, 1, 2, 3},
+			exp:     []uint16{0, 1, 2, 3},
 		},
 	}
 	for i, test := range tests {
@@ -1536,17 +1536,17 @@ func TestDifferenceRunRun(t *testing.T) {
 	a := &container{}
 	b := &container{}
 	tests := []struct {
-		aruns []interval32
-		bruns []interval32
-		exp   []interval32
+		aruns []interval16
+		bruns []interval16
+		exp   []interval16
 	}{
 		{
 			// this tests all six overlap combinations
 			// A                     [   ]                   [   ]                        [ ]              [     ]             [   ]                 [  ]
 			// B                    [     ]                [   ]                     [ ]                     [  ]                [   ]                    [  ]
-			aruns: []interval32{{start: 3, last: 6}, {start: 13, last: 16}, {start: 24, last: 26}, {start: 33, last: 38}, {start: 43, last: 46}, {start: 53, last: 56}},
-			bruns: []interval32{{start: 1, last: 8}, {start: 11, last: 14}, {start: 21, last: 23}, {start: 35, last: 37}, {start: 44, last: 48}, {start: 57, last: 59}},
-			exp:   []interval32{{start: 15, last: 16}, {start: 24, last: 26}, {start: 33, last: 34}, {start: 38, last: 38}, {start: 43, last: 43}, {start: 53, last: 56}},
+			aruns: []interval16{{start: 3, last: 6}, {start: 13, last: 16}, {start: 24, last: 26}, {start: 33, last: 38}, {start: 43, last: 46}, {start: 53, last: 56}},
+			bruns: []interval16{{start: 1, last: 8}, {start: 11, last: 14}, {start: 21, last: 23}, {start: 35, last: 37}, {start: 44, last: 48}, {start: 57, last: 59}},
+			exp:   []interval16{{start: 15, last: 16}, {start: 24, last: 26}, {start: 33, last: 34}, {start: 38, last: 38}, {start: 43, last: 43}, {start: 53, last: 56}},
 		},
 	}
 	for i, test := range tests {
@@ -1562,7 +1562,7 @@ func TestDifferenceRunRun(t *testing.T) {
 }
 
 func TestWriteReadArray(t *testing.T) {
-	ca := &container{array: []uint32{1, 10, 100, 1000}, n: 4}
+	ca := &container{array: []uint16{1, 10, 100, 1000}, n: 4, container_type: ContainerArray}
 	ba := &Bitmap{keys: []uint64{0}, containers: []*container{ca}}
 	ba2 := &Bitmap{}
 	var buf bytes.Buffer
@@ -1581,7 +1581,7 @@ func TestWriteReadArray(t *testing.T) {
 
 func TestWriteReadBitmap(t *testing.T) {
 	// create bitmap containing > 4096 bits
-	cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32}
+	cb := &container{bitmap: make([]uint64, bitmapN), n: 129 * 32, container_type: ContainerBitmap}
 	for i := 0; i < 129; i++ {
 		cb.bitmap[i] = 0x5555555555555555
 	}
@@ -1602,7 +1602,7 @@ func TestWriteReadBitmap(t *testing.T) {
 }
 
 func TestWriteReadRun(t *testing.T) {
-	cr := &container{runs: []interval32{{start: 3, last: 13}, {start: 100, last: 109}}, n: 20}
+	cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 20, container_type: ContainerRun}
 	br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}}
 	br2 := &Bitmap{}
 	var buf bytes.Buffer
@@ -1620,9 +1620,9 @@ func TestWriteReadRun(t *testing.T) {
 }
 
 func TestXorArrayRun(t *testing.T) {
-	a := &container{array: []uint32{1, 5, 10, 11, 12}}
-	b := &container{runs: []interval32{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
-	exp := []uint32{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}
+	a := &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray}
+	b := &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}
+	exp := []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}
 
 	//ret := xorArrayRun(a, b)
 	ret := xor(a, b)
@@ -1634,8 +1634,8 @@ func TestXorArrayRun(t *testing.T) {
 	if !reflect.DeepEqual(ret.array, exp) {
 		t.Fatalf("test #2 expected %v, but got %v", exp, ret.array)
 	}
-	c := &container{array: []uint32{1, 5, 10, 11, 12, 13, 14}}
-	exp = []uint32{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}
+	c := &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray}
+	exp = []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}
 	ret = xor(b, c)
 	if !reflect.DeepEqual(ret.array, exp) {
 		t.Fatalf("test #3 expected %v, but got %v", exp, ret.array)
@@ -1648,97 +1648,97 @@ func TestXorArrayRun(t *testing.T) {
 
 //special case that didn't fit the xorrunrun table testing below.
 func TestXorRunRun1(t *testing.T) {
-	a := &container{}
-	b := &container{}
-	a.runs = []interval32{{start: 4, last: 10}}
-	b.runs = []interval32{{start: 5, last: 10}}
+	a := &container{container_type: ContainerRun}
+	b := &container{container_type: ContainerRun}
+	a.runs = []interval16{{start: 4, last: 10}}
+	b.runs = []interval16{{start: 5, last: 10}}
 	ret := xorRunRun(a, b)
-	if !reflect.DeepEqual(ret.array, []uint32{4}) {
-		t.Fatalf("test #1 expected %v, but got %v", []uint32{4}, ret.array)
+	if !reflect.DeepEqual(ret.array, []uint16{4}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint16{4}, ret.array)
 	}
 	ret = xorRunRun(b, a)
-	if !reflect.DeepEqual(ret.array, []uint32{4}) {
-		t.Fatalf("test #1 expected %v, but got %v", []uint32{4}, ret.array)
+	if !reflect.DeepEqual(ret.array, []uint16{4}) {
+		t.Fatalf("test #1 expected %v, but got %v", []uint16{4}, ret.array)
 	}
 }
 
 func TestXorRunRun(t *testing.T) {
-	a := &container{}
-	b := &container{}
+	a := &container{container_type: ContainerRun}
+	b := &container{container_type: ContainerRun}
 	tests := []struct {
-		aruns []interval32
-		bruns []interval32
-		exp   []interval32
+		aruns []interval16
+		bruns []interval16
+		exp   []interval16
 	}{
 		{
-			aruns: []interval32{},
-			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 5, last: 10}},
+			aruns: []interval16{},
+			bruns: []interval16{{start: 5, last: 10}},
+			exp:   []interval16{{start: 5, last: 10}},
 		},
 		{
-			aruns: []interval32{{start: 0, last: 4}},
-			bruns: []interval32{{start: 6, last: 10}},
-			exp:   []interval32{{start: 0, last: 4}, {start: 6, last: 10}},
+			aruns: []interval16{{start: 0, last: 4}},
+			bruns: []interval16{{start: 6, last: 10}},
+			exp:   []interval16{{start: 0, last: 4}, {start: 6, last: 10}},
 		},
 		{
-			aruns: []interval32{{start: 0, last: 6}},
-			bruns: []interval32{{start: 4, last: 10}},
-			exp:   []interval32{{start: 0, last: 3}, {start: 7, last: 10}},
+			aruns: []interval16{{start: 0, last: 6}},
+			bruns: []interval16{{start: 4, last: 10}},
+			exp:   []interval16{{start: 0, last: 3}, {start: 7, last: 10}},
 		},
 		{
-			aruns: []interval32{{start: 4, last: 10}},
-			bruns: []interval32{{start: 0, last: 6}},
-			exp:   []interval32{{start: 0, last: 3}, {start: 7, last: 10}},
+			aruns: []interval16{{start: 4, last: 10}},
+			bruns: []interval16{{start: 0, last: 6}},
+			exp:   []interval16{{start: 0, last: 3}, {start: 7, last: 10}},
 		},
 		{
-			aruns: []interval32{{start: 0, last: 10}},
-			bruns: []interval32{{start: 0, last: 6}},
-			exp:   []interval32{{start: 7, last: 10}},
+			aruns: []interval16{{start: 0, last: 10}},
+			bruns: []interval16{{start: 0, last: 6}},
+			exp:   []interval16{{start: 7, last: 10}},
 		},
 		{
-			aruns: []interval32{{start: 0, last: 6}},
-			bruns: []interval32{{start: 0, last: 10}},
-			exp:   []interval32{{start: 7, last: 10}},
+			aruns: []interval16{{start: 0, last: 6}},
+			bruns: []interval16{{start: 0, last: 10}},
+			exp:   []interval16{{start: 7, last: 10}},
 		},
 		{
-			aruns: []interval32{{start: 0, last: 6}},
-			bruns: []interval32{{start: 0, last: 10}},
-			exp:   []interval32{{start: 7, last: 10}},
+			aruns: []interval16{{start: 0, last: 6}},
+			bruns: []interval16{{start: 0, last: 10}},
+			exp:   []interval16{{start: 7, last: 10}},
 		},
 		{
-			aruns: []interval32{{start: 5, last: 12}},
-			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 11, last: 12}},
+			aruns: []interval16{{start: 5, last: 12}},
+			bruns: []interval16{{start: 5, last: 10}},
+			exp:   []interval16{{start: 11, last: 12}},
 		},
 		{
-			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
-			bruns: []interval32{{start: 5, last: 10}},
-			exp:   []interval32{{start: 1, last: 3}, {start: 6, last: 6}, {start: 11, last: 12}},
+			aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			bruns: []interval16{{start: 5, last: 10}},
+			exp:   []interval16{{start: 1, last: 3}, {start: 6, last: 6}, {start: 11, last: 12}},
 		},
 		{
-			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
-			bruns: []interval32{{start: 2, last: 65535}},
-			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
+			aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			bruns: []interval16{{start: 2, last: 65535}},
+			exp:   []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
 		},
 		{
-			aruns: []interval32{{start: 2, last: 65535}},
-			bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
-			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
+			aruns: []interval16{{start: 2, last: 65535}},
+			bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			exp:   []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
 		},
 		{
-			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
-			bruns: []interval32{{start: 0, last: 65535}},
-			exp:   []interval32{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
+			aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			bruns: []interval16{{start: 0, last: 65535}},
+			exp:   []interval16{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
 		},
 		{
-			aruns: []interval32{{start: 0, last: 65535}},
-			bruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
-			exp:   []interval32{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
+			aruns: []interval16{{start: 0, last: 65535}},
+			bruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 12}},
+			exp:   []interval16{{start: 0, last: 0}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 13, last: 65535}},
 		},
 		{
-			aruns: []interval32{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}},
-			bruns: []interval32{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}},
-			exp:   []interval32{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}},
+			aruns: []interval16{{start: 1, last: 3}, {start: 5, last: 5}, {start: 7, last: 9}, {start: 12, last: 22}},
+			bruns: []interval16{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}},
+			exp:   []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}},
 		},
 	}
 	for i, test := range tests {
@@ -1756,7 +1756,7 @@ func TestXorRunRun(t *testing.T) {
 }
 
 func TestBitmapXorRange(t *testing.T) {
-	c := &container{bitmap: make([]uint64, bitmapN)}
+	c := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
 	tests := []struct {
 		bitmap []uint64
 		start  uint64
@@ -1824,16 +1824,16 @@ func TestBitmapXorRange(t *testing.T) {
 }
 
 func TestXorBitmapRun(t *testing.T) {
-	a := &container{}
-	b := &container{}
+	a := &container{container_type: ContainerBitmap}
+	b := &container{container_type: ContainerRun}
 	tests := []struct {
 		bitmap []uint64
-		runs   []interval32
+		runs   []interval16
 		exp    []uint64
 	}{
 		{
 			bitmap: []uint64{0x0, 0x0, 0x0},
-			runs:   []interval32{{start: 129, last: 131}},
+			runs:   []interval16{{start: 129, last: 131}},
 			exp:    []uint64{0x0, 0x0, 0x00000000000000E},
 		},
 	}
@@ -2016,40 +2016,40 @@ func TestIteratorRuns(t *testing.T) {
 
 func TestRunBinSearchContains(t *testing.T) {
 	tests := []struct {
-		runs  []interval32
-		index uint32
+		runs  []interval16
+		index uint16
 		exp   struct {
 			index int
 			found bool
 		}
 	}{
 		{
-			runs:  []interval32{{start: 0, last: 10}},
-			index: uint32(3),
+			runs:  []interval16{{start: 0, last: 10}},
+			index: uint16(3),
 			exp: struct {
 				index int
 				found bool
 			}{index: 0, found: true},
 		},
 		{
-			runs:  []interval32{{start: 0, last: 10}},
-			index: uint32(13),
+			runs:  []interval16{{start: 0, last: 10}},
+			index: uint16(13),
 			exp: struct {
 				index int
 				found bool
 			}{index: 0, found: false},
 		},
 		{
-			runs:  []interval32{{start: 0, last: 10}, {start: 20, last: 30}},
-			index: uint32(13),
+			runs:  []interval16{{start: 0, last: 10}, {start: 20, last: 30}},
+			index: uint16(13),
 			exp: struct {
 				index int
 				found bool
 			}{index: 0, found: false},
 		},
 		{
-			runs:  []interval32{{start: 0, last: 10}, {start: 20, last: 30}},
-			index: uint32(36),
+			runs:  []interval16{{start: 0, last: 10}, {start: 20, last: 30}},
+			index: uint16(36),
 			exp: struct {
 				index int
 				found bool
@@ -2069,55 +2069,55 @@ func TestRunBinSearchContains(t *testing.T) {
 
 func TestRunBinSearch(t *testing.T) {
 	tests := []struct {
-		runs   []interval32
-		search uint32
+		runs   []interval16
+		search uint16
 		exp    bool
 		expi   int
 	}{
 		{
-			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			runs:   []interval16{{2, 10}, {50, 60}, {80, 90}},
 			search: 1,
 			exp:    false,
 			expi:   0,
 		},
 		{
-			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			runs:   []interval16{{2, 10}, {50, 60}, {80, 90}},
 			search: 2,
 			exp:    true,
 			expi:   0,
 		},
 		{
-			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			runs:   []interval16{{2, 10}, {50, 60}, {80, 90}},
 			search: 5,
 			exp:    true,
 			expi:   0,
 		},
 		{
-			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			runs:   []interval16{{2, 10}, {50, 60}, {80, 90}},
 			search: 10,
 			exp:    true,
 			expi:   0,
 		},
 		{
-			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			runs:   []interval16{{2, 10}, {50, 60}, {80, 90}},
 			search: 20,
 			exp:    false,
 			expi:   1,
 		},
 		{
-			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			runs:   []interval16{{2, 10}, {50, 60}, {80, 90}},
 			search: 55,
 			exp:    true,
 			expi:   1,
 		},
 		{
-			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			runs:   []interval16{{2, 10}, {50, 60}, {80, 90}},
 			search: 70,
 			exp:    false,
 			expi:   2,
 		},
 		{
-			runs:   []interval32{{2, 10}, {50, 60}, {80, 90}},
+			runs:   []interval16{{2, 10}, {50, 60}, {80, 90}},
 			search: 100,
 			exp:    false,
 			expi:   3,
@@ -2169,16 +2169,16 @@ func Test_BufBitmapIterator_Next(t *testing.T) {
 		t.Fatalf("wrong container type")
 	}
 
-	bin := []uint32{}
+	bin := []uint16{}
 
 	itr := newBufBitmapIterator(newBitmapIterator(b.containers[0].bitmap))
-	x := uint32(0)
+	x := uint16(0)
 
 	for i := 0; i < 10; i++ {
 		x, _ = itr.next()
 		bin = append(bin, x)
 	}
-	exp := []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+	exp := []uint16{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
 	if !reflect.DeepEqual(bin, exp) {
 		t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin)
 	}
@@ -2187,7 +2187,7 @@ func Test_BufBitmapIterator_Next(t *testing.T) {
 	itr.unread()
 	x, _ = itr.next()
 	bin = append(bin, x)
-	exp = append(exp, uint32(9))
+	exp = append(exp, uint16(9))
 	if !reflect.DeepEqual(bin, exp) {
 		t.Fatalf("BufBitmapIterator expected (%v) but got (%v)", exp, bin)
 	}
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index d4f8dc8d7..05857465a 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -46,15 +46,25 @@ func TestBitmapClone(t *testing.T) {
 	}
 }
 
+func TestContainerCount(t *testing.T) {
+	b := roaring.NewBitmap(65535)
+
+	if b.Count() != b.CountRange(0, 65546) {
+		t.Fatalf("Count != CountRange\n")
+	}
+}
+
 func TestCheckBitmap(t *testing.T) {
 	b := roaring.NewBitmap()
+	x := 0
 	for i := uint64(61000); i < 71000; i++ {
+		x++
 		b.Add(i)
 	}
 	for i := uint64(75000); i < 75100; i++ {
+		x++
 		b.Add(i)
 	}
-
 	err := b.Check()
 	if err != nil {
 		t.Fatalf("%v\n", err)

From 6c4f37ae704896a49e16e8ca7510d96bca7daaed Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 28 Jul 2017 13:24:41 -0500
Subject: [PATCH 092/100] 16bit pass 1

---
 roaring/roaring.go | 78 +++++++++++++++++++++++++++++-----------------
 1 file changed, 50 insertions(+), 28 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 4c43e000e..d7ae9b6d3 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -510,15 +510,6 @@ func (b *Bitmap) Optimize() {
 		c.Optimize()
 	}
 }
-func checkA(a []uint16) bool {
-	for _, x := range a {
-		if x > 65535 {
-			return true
-		}
-	}
-	return false
-
-}
 
 // WriteTo writes b to w.
 func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
@@ -533,7 +524,6 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 	// Build header before writing individual container blocks.
 	// Metadata for each container is 4+8+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset)
 	buf := make([]byte, headerSize+(containerCount*(4+4+4+4)))
-
 	// Cookie header section.
 	binary.LittleEndian.PutUint32(buf[0:], thisCookie)
 	binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount))
@@ -550,7 +540,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 		//assert(c.count() == c.n, "cannot write container count, mismatch: count=%d, n=%d", count, c.n)
 		if c.n > 0 {
 			binary.LittleEndian.PutUint64(buf[headerSize+(i-empty)*12:], uint64(key))
-			buf[headerSize+(i-empty)*12+8] = c.container_type
+			binary.LittleEndian.PutUint16(buf[headerSize+(i-empty)*12+8:], uint16(c.container_type))
 			binary.LittleEndian.PutUint16(buf[headerSize+(i-empty)*12+8+2:], uint16(c.n-1))
 		} else {
 			empty++
@@ -565,10 +555,10 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 
 		if c.n > 0 {
 			binary.LittleEndian.PutUint32(buf[headerSize+(containerCount*12)+((i-empty)*4):], uint32(offset))
+			offset += uint32(c.size())
 		} else {
 			empty++
 		}
-		offset += uint32(c.size())
 	}
 
 	// Write header.
@@ -624,7 +614,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 	for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
 		b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
 		b.containers[i] = &container{
-			container_type: buf[8],
+			container_type: byte(binary.LittleEndian.Uint16(buf[8:10])),
 			n:              int(binary.LittleEndian.Uint16(buf[10:12]) + 1),
 			mapped:         true,
 		}
@@ -634,7 +624,6 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 	// Read container offsets and attach data.
 	for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] {
 		offset := binary.LittleEndian.Uint32(buf[0:4])
-
 		// Verify the offset is within the bounds of the input data.
 		if int(offset) >= len(data) {
 			return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data))
@@ -1226,25 +1215,57 @@ func (c *container) arrayCountRuns() (r int) {
 	return r
 }
 
+func (c *container) countRuns() (r int) {
+	if c.isArray() {
+		return c.arrayCountRuns()
+	} else if c.isBitmap() {
+		return c.bitmapCountRuns()
+	} else if c.isRun() {
+		return len(c.runs)
+	}
+
+	// sure hope this never happens
+	return 0
+}
+
 // Optimize converts the container to the type which will take up the least
 // amount of space.
 func (c *container) Optimize() {
+	if c.n == 0 {
+		return
+	}
+	runs := c.countRuns()
+
+	// First decide which type to use.
+	var newType byte
+	if runs <= RunMaxSize && runs <= c.n/2 {
+		newType = ContainerRun
+	} else if c.n < ArrayMaxSize {
+		newType = ContainerArray
+	} else {
+		newType = ContainerBitmap
+	}
+
+	// Then convert accordingly.
 	if c.isArray() {
-		runs := c.arrayCountRuns()
-		if runs < c.n/2 {
+		if newType == ContainerBitmap {
+			c.arrayToBitmap()
+		} else if newType == ContainerRun {
 			c.arrayToRun()
 		}
 	} else if c.isBitmap() {
-		runs := c.bitmapCountRuns()
-		if runs < RunMaxSize {
+		if newType == ContainerArray {
+			c.bitmapToArray()
+		} else if newType == ContainerRun {
 			c.bitmapToRun()
 		}
 	} else if c.isRun() {
-		if len(c.runs) > RunMaxSize {
+		if newType == ContainerBitmap {
 			c.runToBitmap()
+		} else if newType == ContainerArray {
+			c.runToArray()
 		}
 	}
-
 }
 
 func (c *container) arrayContains(v uint16) bool {
@@ -1438,7 +1459,8 @@ func (c *container) runToBitmap() {
 
 	for _, r := range c.runs {
 		// TODO this can be ~64x faster for long runs by setting maxBitmap instead of single bits
-		for v := r.start; v <= r.last; v++ {
+		//note v must be int or will overflow
+		for v := int(r.start); v <= int(r.last); v++ {
 			c.bitmap[int(v)/64] |= (uint64(1) << uint(v%64))
 		}
 	}
@@ -1541,8 +1563,8 @@ func (c *container) runToArray() {
 	}
 
 	for _, r := range c.runs {
-		for v := r.start; v <= r.last; v++ {
-			c.array = append(c.array, v)
+		for v := int(r.start); v <= int(r.last); v++ {
+			c.array = append(c.array, uint16(v))
 		}
 	}
 	c.runs = nil
@@ -2127,7 +2149,7 @@ func unionArrayArray(a, b *container) *container {
 // unionArrayRun optimistically assumes that the result will be a run container,
 // and converts to a bitmap or array container afterwards if necessary.
 func unionArrayRun(a, b *container) *container {
-	if b.n == 65535 {
+	if b.n == maxContainerVal {
 		return b.clone()
 	}
 	output := &container{}
@@ -2169,7 +2191,7 @@ func (c *container) runAppendInterval(v interval16) int {
 		return int(v.last - v.start + 1)
 	} else {
 		last := c.runs[len(c.runs)-1]
-		if last.last == 65535 { //protect against overflow
+		if last.last == maxContainerVal { //protect against overflow
 			return 0
 		}
 		if last.last+1 >= v.start && v.last > last.last {
@@ -2184,10 +2206,10 @@ func (c *container) runAppendInterval(v interval16) int {
 }
 
 func unionRunRun(a, b *container) *container {
-	if a.n == 65535 {
+	if a.n == maxContainerVal {
 		return a.clone()
 	}
-	if b.n == 65535 {
+	if b.n == maxContainerVal {
 		return b.clone()
 	}
 	na, nb := len(a.runs), len(b.runs)
@@ -2217,7 +2239,7 @@ func unionRunRun(a, b *container) *container {
 }
 
 func unionBitmapRun(a, b *container) *container {
-	if b.n == 65535 {
+	if b.n == maxContainerVal {
 		return b.clone()
 	}
 	output := a.clone()

From a3f3ca3c5abc25a8dd10099af4d4fd138a308ae0 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Tue, 1 Aug 2017 16:25:11 -0500
Subject: [PATCH 093/100] fixed overflow in interval16 runlen; added rle tests

---
 roaring/roaring.go               |  9 +++++----
 roaring/roaring_internal_test.go | 18 ++++++++++++++++++
 roaring/roaring_test.go          | 18 ++++++++++++++++++
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index d7ae9b6d3..57aa41e96 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -948,7 +948,7 @@ type interval16 struct {
 
 // runlen returns the count of integers in the interval.
 func (iv interval16) runlen() int {
-	return int(1 + iv.last - iv.start)
+	return 1 + int(iv.last-iv.start)
 }
 
 // newContainer returns a new instance of container.
@@ -958,17 +958,17 @@ func newContainer() *container {
 
 // isArray returns true if the container is an array container.
 func (c *container) isArray() bool {
-	return c.bitmap == nil && c.runs == nil
+	return c.container_type == ContainerArray
 }
 
 // isBitmap returns true if the container is a bitmap container.
 func (c *container) isBitmap() bool {
-	return c.array == nil && c.runs == nil
+	return c.container_type == ContainerBitmap
 }
 
 // isRun returns true if the container is a run-length-encoded container.
 func (c *container) isRun() bool {
-	return c.array == nil && c.bitmap == nil
+	return c.container_type == ContainerRun
 }
 
 // unmap creates copies of the containers data in the heap.
@@ -1507,6 +1507,7 @@ func (c *container) bitmapToRun() {
 		}
 
 		if current == maxBitmap {
+
 			// bitmap[1023] == maxBitmap
 			c.runs = append(c.runs, interval16{start, maxContainerVal})
 			break
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index f1e64068e..f9540d943 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1022,6 +1022,15 @@ func TestRunToBitmap(t *testing.T) {
 	}
 }
 
+func getFullBitmap() []uint64 {
+	x := make([]uint64, 1024, 1024)
+	for i := range x {
+		x[i] = uint64(0xFFFFFFFFFFFFFFFF)
+	}
+	return x
+
+}
+
 func TestBitmapToRun(t *testing.T) {
 	a := &container{}
 	tests := []struct {
@@ -1072,6 +1081,10 @@ func TestBitmapToRun(t *testing.T) {
 			bitmap: make([]uint64, bitmapN),
 			exp:    []interval16{{start: 65408, last: 65535}},
 		},
+		{
+			bitmap: getFullBitmap(),
+			exp:    []interval16{{start: 0, last: 65535}},
+		},
 	}
 	tests[8].bitmap[1022] = 0xFFFFFFFFFFFFFFFF
 	tests[8].bitmap[1023] = 0xFFFFFFFFFFFFFFFF
@@ -1084,10 +1097,15 @@ func TestBitmapToRun(t *testing.T) {
 			n += int(popcount(v))
 		}
 		a.n = n
+		x := a.bitmap
 		a.bitmapToRun()
 		if !reflect.DeepEqual(a.runs, test.exp) {
 			t.Fatalf("test #%v expected %v, but got %v", i, test.exp, a.runs)
 		}
+		a.runToBitmap()
+		if !reflect.DeepEqual(a.bitmap, x) {
+			t.Fatalf("test #%v expected %v, but got %v", i, a.bitmap, x)
+		}
 	}
 }
 
diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go
index 05857465a..8de92eb4e 100644
--- a/roaring/roaring_test.go
+++ b/roaring/roaring_test.go
@@ -87,6 +87,24 @@ func TestCheckRun(t *testing.T) {
 		t.Fatalf("%v\n", err)
 	}
 }
+func TestCheckFullRun(t *testing.T) {
+	b := roaring.NewBitmap()
+	for i := uint64(0); i < 2097152; i++ {
+		if i%16384 == 0 {
+			b.Optimize() // convert to runs
+		}
+		b.Add(i)
+	}
+	err := b.Check()
+	if err != nil {
+		t.Fatalf("Before %v\n", err)
+	}
+	b.Optimize() // convert to runs
+	err = b.Check()
+	if err != nil {
+		t.Fatalf("After %v\n", err)
+	}
+}
 
 // Ensure that we can transition between runs and arrays when materializing the bitmap.
 func TestContainerTransitions(t *testing.T) {

From 091e982263ff11cd29c1d642e7b64a2297d468e2 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Tue, 1 Aug 2017 21:47:28 -0500
Subject: [PATCH 094/100] update tests to support container_type

---
 roaring/roaring.go               | 62 +++++++++++++++++++-------------
 roaring/roaring_internal_test.go | 38 ++++++++++++++------
 2 files changed, 65 insertions(+), 35 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 57aa41e96..464389de6 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -516,8 +516,8 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 	b.Optimize()
 	// Remove empty containers before persisting.
 	//b.removeEmptyContainers()
-	containerCount := len(b.keys) - b.countEmptyContainers()
 
+	containerCount := len(b.keys) - b.countEmptyContainers()
 	thisCookie := cookieNoRuns
 	headerSize := headerBaseSize
 
@@ -1259,12 +1259,22 @@ func (c *container) Optimize() {
 		} else if newType == ContainerRun {
 			c.bitmapToRun()
 		}
+		e := c.check()
+		if e != nil {
+			fmt.Printf("Bitmap %v\n", c)
+			panic(e)
+		}
 	} else if c.isRun() {
 		if newType == ContainerBitmap {
 			c.runToBitmap()
 		} else if newType == ContainerArray {
 			c.runToArray()
 		}
+		e := c.check()
+		if e != nil {
+			fmt.Printf("Run %v\n", c)
+			panic(e)
+		}
 	}
 }
 
@@ -1329,7 +1339,7 @@ func (c *container) bitmapRemove(v uint16) bool {
 
 	// Lower count and remove element.
 	// c.n-- // TODO removed this - test it
-	c.bitmap[v/64] &^= (uint64(1) << (v % 64))
+	c.bitmap[v/64] &^= (uint64(1) << uint(v%64))
 
 	// Convert to array if we go below the threshold.
 	if c.n == ArrayMaxSize {
@@ -1904,7 +1914,7 @@ func intersect(a, b *container) *container {
 }
 
 func intersectArrayArray(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.array[i], b.array[j]
@@ -1925,7 +1935,7 @@ func intersectArrayArray(a, b *container) *container {
 // container. The return is always an array container (since it's guaranteed to
 // be low-cardinality)
 func intersectArrayRun(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	na, nb := len(a.array), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.array[i], b.runs[j]
@@ -1944,7 +1954,7 @@ func intersectArrayRun(a, b *container) *container {
 
 // intersectRunRun computes the intersect of two run containers.
 func intersectRunRun(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerRun}
 	na, nb := len(a.runs), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.runs[i], b.runs[j]
@@ -1986,7 +1996,7 @@ func intersectBitmapRun(a, b *container) *container {
 	var output *container
 	if b.n < ArrayMaxSize {
 		// output is array container
-		output = &container{}
+		output = &container{container_type: ContainerArray}
 		for _, iv := range b.runs {
 			for i := iv.start; i <= iv.last; i++ {
 				if a.bitmapContains(i) {
@@ -2000,7 +2010,8 @@ func intersectBitmapRun(a, b *container) *container {
 		// bitmap that are in the runs. alternately, we could zero out ranges in
 		// the bitmap which are between runs.
 		output = &container{
-			bitmap: make([]uint64, bitmapN),
+			bitmap:         make([]uint64, bitmapN),
+			container_type: ContainerBitmap,
 		}
 		for j := 0; j < len(b.runs); j++ {
 			vb := b.runs[j]
@@ -2041,7 +2052,7 @@ func intersectBitmapRun(a, b *container) *container {
 }
 
 func intersectArrayBitmap(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	itra := newArrayIterator(a.array)
 	itrb := newBitmapIterator(b.bitmap)
 	va, eof1 := itra.next()
@@ -2065,7 +2076,7 @@ func intersectArrayBitmap(a, b *container) *container {
 }
 
 func intersectBitmapBitmap(a, b *container) *container {
-	output := &container{}
+	output := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
 	itr0 := newBitmapIterator(a.bitmap)
 	itr1 := newBitmapIterator(b.bitmap)
 	va, eof1 := itr0.next()
@@ -2117,7 +2128,7 @@ func union(a, b *container) *container {
 }
 
 func unionArrayArray(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; ; {
 		if i >= na && j >= nb {
@@ -2153,7 +2164,7 @@ func unionArrayRun(a, b *container) *container {
 	if b.n == maxContainerVal {
 		return b.clone()
 	}
-	output := &container{}
+	output := &container{container_type: ContainerRun}
 	na, nb := len(a.array), len(b.runs)
 	var vb interval16
 	var va uint16
@@ -2189,7 +2200,7 @@ func unionArrayRun(a, b *container) *container {
 func (c *container) runAppendInterval(v interval16) int {
 	if len(c.runs) == 0 {
 		c.runs = append(c.runs, v)
-		return int(v.last - v.start + 1)
+		return int(v.last-v.start) + 1
 	} else {
 		last := c.runs[len(c.runs)-1]
 		if last.last == maxContainerVal { //protect against overflow
@@ -2200,7 +2211,7 @@ func (c *container) runAppendInterval(v interval16) int {
 			return int(v.last - last.last)
 		} else if last.last+1 < v.start {
 			c.runs = append(c.runs, v)
-			return int(v.last - v.start + 1)
+			return int(v.last-v.start) + 1
 		}
 	}
 	return 0
@@ -2215,7 +2226,8 @@ func unionRunRun(a, b *container) *container {
 	}
 	na, nb := len(a.runs), len(b.runs)
 	output := &container{
-		runs: make([]interval16, 0, na+nb),
+		runs:           make([]interval16, 0, na+nb),
+		container_type: ContainerRun,
 	}
 	var va, vb interval16
 	for i, j := 0, 0; i < na || j < nb; {
@@ -2322,7 +2334,7 @@ func (c *container) bitmapZeroRange(i, j uint64) {
 }
 
 func unionArrayBitmap(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	itr := newBufBitmapIterator(newBitmapIterator(b.bitmap))
 	for i := 0; ; {
 		vb, eof := itr.next()
@@ -2354,7 +2366,8 @@ func unionArrayBitmap(a, b *container) *container {
 
 func unionBitmapBitmap(a, b *container) *container {
 	output := &container{
-		bitmap: make([]uint64, bitmapN),
+		bitmap:         make([]uint64, bitmapN),
+		container_type: ContainerBitmap,
 	}
 
 	for i := 0; i < bitmapN; i++ {
@@ -2396,7 +2409,7 @@ func difference(a, b *container) *container {
 
 // differenceArrayArray computes the difference bween two arrays.
 func differenceArrayArray(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; i < na; {
 		va := a.array[i]
@@ -2427,7 +2440,7 @@ func differenceArrayRun(a, b *container) *container {
 		return a.clone()
 	}
 
-	output := &container{array: make([]uint16, 0, a.n)}
+	output := &container{array: make([]uint16, 0, a.n), container_type: ContainerArray}
 	// cardinality upper bound: card(A)
 
 	i := 0 // array index
@@ -2618,7 +2631,7 @@ func differenceRunRun(a, b *container) *container {
 }
 
 func differenceArrayBitmap(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	itr := newBufBitmapIterator(newBitmapIterator(b.bitmap))
 	for i := 0; i < len(a.array); {
 		va := a.array[i]
@@ -2643,7 +2656,7 @@ func differenceArrayBitmap(a, b *container) *container {
 }
 
 func differenceBitmapArray(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	itr := newBufBitmapIterator(newBitmapIterator(a.bitmap))
 	i := 0
 	va, eof := itr.next()
@@ -2673,7 +2686,7 @@ func differenceBitmapArray(a, b *container) *container {
 }
 
 func differenceBitmapBitmap(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	itr0 := newBufBitmapIterator(newBitmapIterator(a.bitmap))
 	itr1 := newBufBitmapIterator(newBitmapIterator(b.bitmap))
 	v0, eof0 := itr0.next()
@@ -2729,7 +2742,7 @@ func xor(a, b *container) *container {
 }
 
 func xorArrayArray(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerArray}
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; i < na || j < nb; {
 		if i < na && j >= nb {
@@ -2776,7 +2789,8 @@ func xorArrayBitmap(a, b *container) *container {
 
 func xorBitmapBitmap(a, b *container) *container {
 	output := &container{
-		bitmap: make([]uint64, bitmapN),
+		bitmap:         make([]uint64, bitmapN),
+		container_type: ContainerBitmap,
 	}
 	for i := 0; i < bitmapN; i++ {
 		v := a.bitmap[i] ^ b.bitmap[i]
@@ -3117,7 +3131,7 @@ func assert(condition bool, format string, a ...interface{}) {
 
 // xorArrayRun computes the exclusive or of an array and a run container.
 func xorArrayRun(a, b *container) *container {
-	output := &container{}
+	output := &container{container_type: ContainerRun}
 	na, nb := len(a.array), len(b.runs)
 	var vb interval16
 	var va uint16
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index f9540d943..cf2c51205 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -271,7 +271,7 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
 }
 
 func TestRunRemove(t *testing.T) {
-	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}
 	tests := []struct {
 		op     uint16
 		exp    []interval16
@@ -305,7 +305,7 @@ func TestRunRemove(t *testing.T) {
 }
 
 func TestRunMax(t *testing.T) {
-	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}}
+	c := container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}
 	max := c.max()
 	if max != 16 {
 		t.Fatalf("max for %v should be 16", c.runs)
@@ -621,8 +621,11 @@ func TestUnionMixed(t *testing.T) {
 	c := &container{}
 
 	a.runs = []interval16{{start: 5, last: 10}}
+	a.container_type = ContainerRun
 	a.n = 6
+
 	b.array = []uint16{1, 4, 5, 7, 10, 11, 12}
+	b.container_type = ContainerArray
 	b.n = 7
 	res := union(a, b)
 	if !reflect.DeepEqual(res.array, []uint16{1, 4, 5, 6, 7, 8, 9, 10, 11, 12}) {
@@ -639,6 +642,8 @@ func TestUnionMixed(t *testing.T) {
 	}
 	c.bitmap = []uint64{0x3}
 	c.n = 2
+	c.container_type = ContainerBitmap
+
 	res = union(c, a)
 	if !reflect.DeepEqual(res.bitmap, []uint64{2019}) {
 		t.Fatalf("test #4 expected %v, but got %v", []uint64{2019}, res.bitmap)
@@ -647,6 +652,7 @@ func TestUnionMixed(t *testing.T) {
 	if !reflect.DeepEqual(res.bitmap, []uint64{2019}) {
 		t.Fatalf("test #5 expected %v, but got %v", []uint64{2019}, res.bitmap)
 	}
+
 	res = union(b, c)
 	if !reflect.DeepEqual(res.array, []uint16{0, 1, 4, 5, 7, 10, 11, 12}) {
 		t.Fatalf("test #6 expected %v, but got %v", []uint16{0, 1, 4, 5, 7, 10, 11, 12}, res.array)
@@ -664,8 +670,10 @@ func TestIntersectMixed(t *testing.T) {
 
 	a.runs = []interval16{{start: 5, last: 10}}
 	a.n = 6
+	a.container_type = ContainerRun
 	b.array = []uint16{1, 4, 5, 7, 10, 11, 12}
 	b.n = 7
+	b.container_type = ContainerArray
 	res := intersect(a, b)
 	if !reflect.DeepEqual(res.array, []uint16{5, 7, 10}) {
 		t.Fatalf("test #1 expected %v, but got %v", []uint16{5, 7, 10}, res.array)
@@ -681,6 +689,8 @@ func TestIntersectMixed(t *testing.T) {
 	}
 	c.bitmap = []uint64{0x60}
 	c.n = 2
+	c.container_type = ContainerBitmap
+
 	res = intersect(c, a)
 	if !reflect.DeepEqual(res.array, []uint16{5, 6}) {
 		t.Fatalf("test #4 expected %v, but got %v", []uint16{6}, res.array)
@@ -709,12 +719,15 @@ func TestDifferenceMixed(t *testing.T) {
 
 	a.runs = []interval16{{start: 5, last: 10}}
 	a.n = a.runCountRange(0, 100)
+	a.container_type = ContainerRun
 
 	b.array = []uint16{0, 2, 4, 6, 8, 10, 12}
 	b.n = len(b.array)
+	b.container_type = ContainerArray
 
 	d.array = []uint16{1, 3, 5, 7, 9, 11, 12}
 	d.n = len(d.array)
+	d.container_type = ContainerArray
 
 	res := difference(a, b)
 
@@ -734,6 +747,7 @@ func TestDifferenceMixed(t *testing.T) {
 
 	c.bitmap = []uint64{0x64}
 	c.n = c.countRange(0, 100)
+	c.container_type = ContainerBitmap
 	res = difference(c, a)
 	if !reflect.DeepEqual(res.bitmap, []uint64{0x4}) {
 		t.Fatalf("test #4 expected %v, but got %v", []uint16{4}, res.bitmap)
@@ -1522,8 +1536,8 @@ func TestDifferenceBitmapArray(t *testing.T) {
 	}
 }
 func TestDifferenceBitmapBitmap(t *testing.T) {
-	a := &container{bitmap: make([]uint64, bitmapN)}
-	b := &container{bitmap: make([]uint64, bitmapN)}
+	a := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
+	b := &container{bitmap: make([]uint64, bitmapN), container_type: ContainerBitmap}
 	tests := []struct {
 		abitmap []uint64
 		bbitmap []uint64
@@ -1543,6 +1557,7 @@ func TestDifferenceBitmapBitmap(t *testing.T) {
 	for i, test := range tests {
 		a.bitmap = test.abitmap
 		b.bitmap = test.bbitmap
+
 		ret := differenceBitmapBitmap(a, b)
 		if !reflect.DeepEqual(ret.array, test.exp) {
 			t.Fatalf("test #%v expected \n%X, but got \n%X", i, test.exp, ret.array)
@@ -1551,8 +1566,8 @@ func TestDifferenceBitmapBitmap(t *testing.T) {
 }
 
 func TestDifferenceRunRun(t *testing.T) {
-	a := &container{}
-	b := &container{}
+	a := &container{container_type: ContainerRun}
+	b := &container{container_type: ContainerRun}
 	tests := []struct {
 		aruns []interval16
 		bruns []interval16
@@ -1620,7 +1635,7 @@ func TestWriteReadBitmap(t *testing.T) {
 }
 
 func TestWriteReadRun(t *testing.T) {
-	cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 20, container_type: ContainerRun}
+	cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, container_type: ContainerRun}
 	br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}}
 	br2 := &Bitmap{}
 	var buf bytes.Buffer
@@ -1653,13 +1668,14 @@ func TestXorArrayRun(t *testing.T) {
 		t.Fatalf("test #2 expected %v, but got %v", exp, ret.array)
 	}
 	c := &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray}
-	exp = []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}
+	//	exp = []int16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}
+	expr := []interval16{{start: 1, last: 4}, {start: 6, last: 9}, {start: 11, last: 11}, {start: 14, last: 16}}
 	ret = xor(b, c)
-	if !reflect.DeepEqual(ret.array, exp) {
-		t.Fatalf("test #3 expected %v, but got %v", exp, ret.array)
+	if !reflect.DeepEqual(ret.runs, expr) {
+		t.Fatalf("test #3 expected %v, but got %v", exp, ret.runs)
 	}
 	ret = xor(c, b)
-	if !reflect.DeepEqual(ret.array, exp) {
+	if !reflect.DeepEqual(ret.runs, expr) {
 		t.Fatalf("test #4 expected %v, but got %v", exp, ret.array)
 	}
 }

From 6dacd287b44dbcc4e6a76ae5ab297ed96ffd3c63 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Wed, 2 Aug 2017 10:05:01 -0500
Subject: [PATCH 095/100] fixed overflow on read

---
 roaring/roaring.go               | 13 +------------
 roaring/roaring_internal_test.go | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 464389de6..6dfd110ca 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -615,7 +615,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
 		b.containers[i] = &container{
 			container_type: byte(binary.LittleEndian.Uint16(buf[8:10])),
-			n:              int(binary.LittleEndian.Uint16(buf[10:12]) + 1),
+			n:              int(binary.LittleEndian.Uint16(buf[10:12])) + 1,
 			mapped:         true,
 		}
 	}
@@ -1236,7 +1236,6 @@ func (c *container) Optimize() {
 	}
 	runs := c.countRuns()
 
-	// First decide which type to use.
 	var newType byte
 	if runs <= RunMaxSize && runs <= c.n/2 {
 		newType = ContainerRun
@@ -1259,22 +1258,12 @@ func (c *container) Optimize() {
 		} else if newType == ContainerRun {
 			c.bitmapToRun()
 		}
-		e := c.check()
-		if e != nil {
-			fmt.Printf("Bitmap %v\n", c)
-			panic(e)
-		}
 	} else if c.isRun() {
 		if newType == ContainerBitmap {
 			c.runToBitmap()
 		} else if newType == ContainerArray {
 			c.runToArray()
 		}
-		e := c.check()
-		if e != nil {
-			fmt.Printf("Run %v\n", c)
-			panic(e)
-		}
 	}
 }
 
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index cf2c51205..89741b0e3 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -1634,6 +1634,35 @@ func TestWriteReadBitmap(t *testing.T) {
 	}
 }
 
+func TestWriteReadFullBitmap(t *testing.T) {
+	// create bitmap containing > 4096 bits
+	cb := &container{bitmap: make([]uint64, bitmapN), n: 65536, container_type: ContainerBitmap}
+	for i := 0; i < bitmapN; i++ {
+		cb.bitmap[i] = 0xffffffffffffffff
+	}
+	bb := &Bitmap{keys: []uint64{0}, containers: []*container{cb}}
+	bb2 := &Bitmap{}
+	var buf bytes.Buffer
+	_, err := bb.WriteTo(&buf)
+	if err != nil {
+		t.Fatalf("error writing: %v", err)
+	}
+	err = bb2.UnmarshalBinary(buf.Bytes())
+	if err != nil {
+		t.Fatalf("error unmarshaling: %v", err)
+	}
+	if !reflect.DeepEqual(bb2.containers[0].bitmap, cb.bitmap) {
+		t.Fatalf("bitmap test expected %x, but got %x", cb.bitmap, bb2.containers[0].bitmap)
+	}
+
+	if bb2.containers[0].n != cb.n {
+		t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.containers[0].n)
+	}
+	if bb2.containers[0].count() != cb.count() {
+		t.Fatalf("bitmap test expected count %x, but got %x", cb.n, bb2.containers[0].n)
+	}
+}
+
 func TestWriteReadRun(t *testing.T) {
 	cr := &container{runs: []interval16{{start: 3, last: 13}, {start: 100, last: 109}}, n: 21, container_type: ContainerRun}
 	br := &Bitmap{keys: []uint64{0}, containers: []*container{cr}}

From f6d63ddccb7600e055bc9592fd4c284303c369bf Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Wed, 2 Aug 2017 16:06:58 -0500
Subject: [PATCH 096/100] overflow on intersectBitmapRun

---
 roaring/roaring.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 6dfd110ca..a9284139c 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2007,7 +2007,7 @@ func intersectBitmapRun(a, b *container) *container {
 			i := vb.start >> 6 // index into a
 			vastart := i << 6
 			valast := vastart + 63
-			for valast >= vb.start && vastart <= vb.last {
+			for valast >= vb.start && vastart <= vb.last && i < bitmapN {
 				if vastart >= vb.start && valast <= vb.last { // a within b
 					output.bitmap[i] = a.bitmap[i]
 					output.n += int(popcnt(a.bitmap[i]))

From fbdce18f70bf0369c2370e9c58140f1930fc45bb Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Thu, 3 Aug 2017 14:30:37 -0500
Subject: [PATCH 097/100] overflow in intersectionCountRunRun

---
 roaring/roaring.go               | 33 +++++++++++++++---------------
 roaring/roaring_internal_test.go | 35 +++++++++++++++++++++++++++++---
 2 files changed, 48 insertions(+), 20 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index a9284139c..65b47ca7e 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -336,7 +336,7 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
 		} else if ki > kj {
 			j++
 		} else {
-			n += intersectionCount(b.containers[i], other.containers[j])
+			n += uint64(intersectionCount(b.containers[i], other.containers[j]))
 			i, j = i+1, j+1
 		}
 	}
@@ -1717,7 +1717,7 @@ type ContainerInfo struct {
 	Pointer unsafe.Pointer // offset within the mmap
 }
 
-func intersectionCount(a, b *container) uint64 {
+func intersectionCount(a, b *container) int {
 	if a.isArray() {
 		if b.isArray() {
 			return intersectionCountArrayArray(a, b)
@@ -1745,7 +1745,7 @@ func intersectionCount(a, b *container) uint64 {
 	}
 }
 
-func intersectionCountArrayArray(a, b *container) (n uint64) {
+func intersectionCountArrayArray(a, b *container) (n int) {
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.array[i], b.array[j]
@@ -1761,7 +1761,7 @@ func intersectionCountArrayArray(a, b *container) (n uint64) {
 	return n
 }
 
-func intersectionCountArrayRun(a, b *container) (n uint64) {
+func intersectionCountArrayRun(a, b *container) (n int) {
 	na, nb := len(a.array), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.array[i], b.runs[j]
@@ -1777,8 +1777,7 @@ func intersectionCountArrayRun(a, b *container) (n uint64) {
 	return n
 }
 
-func intersectionCountRunRun(a, b *container) uint64 {
-	var n uint16
+func intersectionCountRunRun(a, b *container) (n int) {
 	na, nb := len(a.runs), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.runs[i], b.runs[j]
@@ -1790,28 +1789,28 @@ func intersectionCountRunRun(a, b *container) uint64 {
 			j++
 		} else if va.last > vb.last && va.start >= vb.start {
 			// |--vb-|-|-va--|
-			n += 1 + vb.last - va.start
+			n += 1 + int(vb.last-va.start)
 			j++
 		} else if va.last > vb.last && va.start < vb.start {
 			// |--va|--vb--|--|
-			n += 1 + vb.last - vb.start
+			n += 1 + int(vb.last-vb.start)
 			j++
 		} else if va.last <= vb.last && va.start >= vb.start {
 			// |--vb|--va--|--|
-			n += 1 + va.last - va.start
+			n += 1 + int(va.last-va.start)
 			i++
 		} else if va.last <= vb.last && va.start < vb.start {
 			// |--va-|-|-vb--|
-			n += 1 + va.last - vb.start
+			n += 1 + int(va.last-vb.start)
 			i++
 		}
 	}
-	return uint64(n)
+	return
 }
 
-func intersectionCountBitmapRun(a, b *container) (n uint64) {
+func intersectionCountBitmapRun(a, b *container) (n int) {
 	for _, iv := range b.runs {
-		n += uint64(a.bitmapCountRange(int(iv.start), int(iv.last)+1))
+		n += a.bitmapCountRange(int(iv.start), int(iv.last)+1)
 	}
 	return n
 }
@@ -1858,20 +1857,20 @@ func intersectionCountArrayBitmapOld(a, b *container) (n uint64) {
 	return n
 }
 
-func intersectionCountArrayBitmap(a, b *container) (n uint64) {
+func intersectionCountArrayBitmap(a, b *container) (n int) {
 	for _, val := range a.array {
 		i := val >> 6
 		if i >= uint16(len(b.bitmap)) {
 			break
 		}
 		off := val % 64
-		n += (b.bitmap[i] & (1 << off)) >> off
+		n += int((b.bitmap[i] & (1 << off)) >> off)
 	}
 	return n
 }
 
-func intersectionCountBitmapBitmap(a, b *container) (n uint64) {
-	return popcntAndSlice(a.bitmap, b.bitmap)
+func intersectionCountBitmapBitmap(a, b *container) (n int) {
+	return int(popcntAndSlice(a.bitmap, b.bitmap))
 }
 
 func intersect(a, b *container) *container {
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 89741b0e3..ea4e5f0ef 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -225,12 +225,39 @@ func TestBitmapCountRange(t *testing.T) {
 	}
 }
 
+func TestIntersectionCountArrayBitmap3(t *testing.T) {
+	a, b := &container{}, &container{}
+	a.container_type = ContainerBitmap
+	a.bitmap = getFullBitmap()
+	a.n = maxContainerVal + 1
+
+	b.container_type = ContainerBitmap
+	b.bitmap = getFullBitmap()
+	b.n = maxContainerVal + 1
+	res := intersectBitmapBitmap(a, b)
+	if res.n != res.count() || res.n != maxContainerVal+1 {
+		t.Fatalf("test #1 intersectCountBitmapBitmap fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1)
+	}
+
+	a.bitmapToRun()
+	res = intersectBitmapRun(b, a)
+	if res.n != res.count() || res.n != maxContainerVal+1 {
+		t.Fatalf("test #2 intersectCountBitmapRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1)
+	}
+	b.bitmapToRun()
+	res = intersectRunRun(a, b)
+	n := intersectionCountRunRun(a, b)
+	if res.n != res.count() || res.n != maxContainerVal+1 || res.n != int(n) {
+		t.Fatalf("test #3 intersectCountRunRun fail orig: %v new: %v exp: %v", res.n, res.count(), maxContainerVal+1)
+	}
+}
+
 func TestIntersectionCountArrayBitmap2(t *testing.T) {
 	a, b := &container{}, &container{}
 	tests := []struct {
 		array  []uint16
 		bitmap []uint64
-		exp    uint64
+		exp    int
 	}{
 		{
 			array:  []uint16{0},
@@ -261,8 +288,10 @@ func TestIntersectionCountArrayBitmap2(t *testing.T) {
 
 	for i, test := range tests {
 		a.array = test.array
+		a.container_type = ContainerArray
 		b.bitmap = test.bitmap
-		ret1 := intersectionCountArrayBitmapOld(a, b)
+		b.container_type = ContainerBitmap
+		ret1 := int(intersectionCountArrayBitmapOld(a, b))
 		ret2 := intersectionCountArrayBitmap(a, b)
 		if ret1 != ret2 || ret2 != test.exp {
 			t.Fatalf("test #%v intersectCountArrayBitmap fail orig: %v new: %v exp: %v", i, ret1, ret2, test.exp)
@@ -352,7 +381,7 @@ func TestIntersectionCountRunRun(t *testing.T) {
 	tests := []struct {
 		aruns []interval16
 		bruns []interval16
-		exp   uint64
+		exp   int
 	}{
 		{
 			aruns: []interval16{},

From af3714e5abca203986f206cf45b9c44131cfac0b Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 4 Aug 2017 10:40:35 -0500
Subject: [PATCH 098/100] corrected comment to reflect reality

---
 roaring/roaring.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index 65b47ca7e..b555e4acf 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -522,8 +522,8 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
 	headerSize := headerBaseSize
 
 	// Build header before writing individual container blocks.
-	// Metadata for each container is 4+8+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset)
-	buf := make([]byte, headerSize+(containerCount*(4+4+4+4)))
+	// Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset)
+	buf := make([]byte, headerSize+(containerCount*(8+2+2+4)))
 	// Cookie header section.
 	binary.LittleEndian.PutUint32(buf[0:], thisCookie)
 	binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount))

From 51f980f69577d5abd5353f6ae6cab82e9907c94b Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Sat, 5 Aug 2017 17:12:31 -0500
Subject: [PATCH 099/100] replaced counting method on cache load

---
 fragment.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/fragment.go b/fragment.go
index 2b923507b..bd29352e7 100644
--- a/fragment.go
+++ b/fragment.go
@@ -272,8 +272,7 @@ func (f *Fragment) openCache() error {
 	// Read in all rows by ID.
 	// This will cause them to be added to the cache.
 	for _, id := range pb.IDs {
-		n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth)
-		//n := f.row(id, true, true).Count()
+		n := f.row(id, false, false).Count()
 		f.cache.BulkAdd(id, n)
 	}
 	f.cache.Invalidate()

From dfbfc217d9ead6c04d0222a7ff4d0b1aea482eaa Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Tue, 8 Aug 2017 12:06:06 -0500
Subject: [PATCH 100/100] corrected some merge issues

---
 fragment_test.go | 2 +-
 server.go        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fragment_test.go b/fragment_test.go
index 8e0a6c749..702eec70a 100644
--- a/fragment_test.go
+++ b/fragment_test.go
@@ -940,7 +940,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
 }
 
 func TestFragment_Snapshot_Run(t *testing.T) {
-    f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+    f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
     defer f.Close()
 
     // Set bits on the fragment.
diff --git a/server.go b/server.go
index 419c496e0..97883fae4 100644
--- a/server.go
+++ b/server.go
@@ -165,7 +165,7 @@ func (s *Server) Open() error {
 	go func() {
 		err := http.Serve(ln, s.Handler)
 		if err != nil {
-			s.logger().Printf("HTTP handler terminated with error: %s\n", err)
+			s.Logger().Printf("HTTP handler terminated with error: %s\n", err)
 		}
 	}()