From a8b070716d7391aa10bf2ecfc88cfd86212e225b Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Wed, 6 Sep 2017 14:41:14 -0500
Subject: [PATCH 1/3] reuse fragment memory

---
 fragment.go        | 3 ++-
 roaring/roaring.go | 7 ++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/fragment.go b/fragment.go
index 829b4e03b..8d68d08a1 100644
--- a/fragment.go
+++ b/fragment.go
@@ -312,7 +312,8 @@ func (f *Fragment) close() error {
 
 func (f *Fragment) closeStorage() error {
 	// Clear the storage bitmap so it doesn't access the closed mmap.
-	f.storage = roaring.NewBitmap()
+
+	//f.storage = roaring.NewBitmap()
 
 	// Unmap the file.
 	if f.storageData != nil {
diff --git a/roaring/roaring.go b/roaring/roaring.go
index 456c9fccc..54e417744 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -617,9 +617,10 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 
 	// Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)).
 	keyN := binary.LittleEndian.Uint32(data[4:8])
-	b.keys = make([]uint64, keyN)
-	b.containers = make([]*container, keyN)
-
+	if int(keyN) != len(b.keys) {
+		b.keys = make([]uint64, keyN)
+		b.containers = make([]*container, keyN)
+	}
 	headerSize := headerBaseSize
 
 	// Descriptive header section: Read container keys and cardinalities.

From 1eb2f8d568ea038dd324b70b7ca8ff30dad39515 Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Thu, 7 Sep 2017 12:59:09 -0500
Subject: [PATCH 2/3] reusing containers for memory efficiency

---
 fragment.go        |  5 +++--
 roaring/roaring.go | 31 +++++++++++++++++++++++--------
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/fragment.go b/fragment.go
index 8d68d08a1..4fa13e70e 100644
--- a/fragment.go
+++ b/fragment.go
@@ -187,8 +187,9 @@ func (f *Fragment) Open() error {
 // openStorage opens the storage bitmap.
 func (f *Fragment) openStorage() error {
 	// Create a roaring bitmap to serve as storage for the slice.
-	f.storage = roaring.NewBitmap()
-
+	if f.storage == nil {
+		f.storage = roaring.NewBitmap()
+	}
 	// Open the data file to be mmap'd and used as an ops log.
 	file, err := os.OpenFile(f.path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
 	if err != nil {
diff --git a/roaring/roaring.go b/roaring/roaring.go
index 54e417744..cab2c4564 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -617,19 +617,34 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 
 	// Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)).
 	keyN := binary.LittleEndian.Uint32(data[4:8])
-	if int(keyN) != len(b.keys) {
-		b.keys = make([]uint64, keyN)
-		b.containers = make([]*container, keyN)
+
+	if len(b.keys) == 0 {
+		b.keys = make([]uint64, 0, keyN)
+		b.containers = make([]*container, 0, keyN)
+	} else if int(keyN) < len(b.keys) { //shrink
+		b.keys = b.keys[:keyN]
+		b.containers = b.containers[:keyN]
 	}
+
 	headerSize := headerBaseSize
 
 	// Descriptive header section: Read container keys and cardinalities.
 	for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] {
-		b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
-		b.containers[i] = &container{
-			container_type: byte(binary.LittleEndian.Uint16(buf[8:10])),
-			n:              int(binary.LittleEndian.Uint16(buf[10:12])) + 1,
-			mapped:         true,
+		// Reuse memory if possible
+		if i >= len(b.keys) {
+			b.keys = append(b.keys, binary.LittleEndian.Uint64(buf[0:8]))
+			b.containers = append(b.containers, &container{
+				container_type: byte(binary.LittleEndian.Uint16(buf[8:10])),
+				n:              int(binary.LittleEndian.Uint16(buf[10:12])) + 1,
+				mapped:         true,
+			})
+		} else {
+			b.keys[i] = binary.LittleEndian.Uint64(buf[0:8])
+			c := b.containers[i]
+			c.container_type = byte(binary.LittleEndian.Uint16(buf[8:10]))
+			c.n = int(binary.LittleEndian.Uint16(buf[10:12])) + 1
+			c.mapped = true
+
 		}
 	}
 	opsOffset := headerSize + int(keyN)*12

From 641519acf2ece69965dd21535b5837dbdf7784cd Mon Sep 17 00:00:00 2001
From: Todd Gruben <tgruben@umbel.com>
Date: Fri, 15 Sep 2017 22:10:48 -0500
Subject: [PATCH 3/3] cleanup on unmarshall shrink

---
 roaring/roaring.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/roaring/roaring.go b/roaring/roaring.go
index cab2c4564..f54efb76f 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -622,6 +622,10 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		b.keys = make([]uint64, 0, keyN)
 		b.containers = make([]*container, 0, keyN)
 	} else if int(keyN) < len(b.keys) { //shrink
+		// nil out to allow to be GCed
+		for i := range b.containers[keyN:] {
+			b.containers[int(keyN)+i] = nil
+		}
 		b.keys = b.keys[:keyN]
 		b.containers = b.containers[:keyN]
 	}