From 6c8e2e9450d65b5bee7874588b984ddf8398aad5 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Tue, 21 Jul 2020 14:33:34 -0500
Subject: [PATCH 01/14] Add test

---
 translator_test.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)

diff --git a/translator_test.go b/translator_test.go
index 72a4e8ddd..c78849357 100644
--- a/translator_test.go
+++ b/translator_test.go
@@ -22,6 +22,7 @@ import (
 	"io"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/pilosa/pilosa/v2"
@@ -296,6 +297,95 @@ func TestTranslation_Reset(t *testing.T) {
 	})
 }
 
+func TestTranslation_Replication(t *testing.T) {
+	t.Run("Replication", func(t *testing.T) {
+		c := test.MustRunCluster(t, 3,
+			[]server.CommandOption{
+				server.OptCommandServerOptions(
+					pilosa.OptServerIsCoordinator(true),
+					pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+					pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+					pilosa.OptServerReplicaN(2),
+				)},
+			[]server.CommandOption{
+				server.OptCommandServerOptions(
+					pilosa.OptServerIsCoordinator(false),
+					pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+					pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+					pilosa.OptServerReplicaN(2),
+				)},
+			[]server.CommandOption{
+				server.OptCommandServerOptions(
+					pilosa.OptServerIsCoordinator(false),
+					pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
+					pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
+					pilosa.OptServerReplicaN(2),
+				)},
+		)
+
+		node0 := c[0]
+		node1 := c[1]
+		//node2 := c[2]
+
+		ctx := context.Background()
+		idx := "i"
+		field := "f"
+
+		// Create an index with keys.
+		if _, err := node0.API.CreateIndex(ctx, idx,
+			pilosa.IndexOptions{
+				Keys: true,
+			}); err != nil {
+			t.Fatal(err)
+		}
+
+		if _, err := node0.API.CreateField(ctx, idx, field); err != nil {
+			t.Fatal(err)
+		}
+
+		// Write data on first node.
+		// these keys are a minimal example to reproduce the problem for the case of a 3-node cluster with replication factor 2
+		if _, err := node0.Queryf(t, idx, "", `
+	    Set("x1", f=1)
+	    Set("x2", f=1)
+    `); err != nil {
+			t.Fatal(err)
+		}
+
+		//exp := `{"results":[{"attrs":{},"columns":[],"keys":["x8","x9","x1","x2","x3","x4","x5","x6","x7"]}]}`
+		exp := `{"results":[{"attrs":{},"columns":[],"keys":["x1","x2"]}]}`
+
+		if !checkClusterState(node0, pilosa.ClusterStateNormal, 1000) {
+			t.Fatalf("unexpected node0 cluster state: %s", node0.API.State())
+		} else if !checkClusterState(node1, pilosa.ClusterStateNormal, 1000) {
+			t.Fatalf("unexpected node1 cluster state: %s", node1.API.State())
+		}
+
+		// Verify the data exists
+		node0.QueryExpect(t, idx, "", `Row(f=1)`, exp)
+
+		// Kill one node.
+		if err := node1.Command.Close(); err != nil {
+			t.Fatal(err)
+		}
+
+		// Verify the data exists with one node down
+		node0.QueryExpect(t, idx, "", `Row(f=1)`, exp)
+	})
+}
+
+// checkClusterState polls a given cluster for its state until it
+// receives a matching state. It polls up to n times before returning.
+func checkClusterState(m *test.Command, state string, n int) bool {
+	for i := 0; i < n; i++ {
+		if m.API.State() == state {
+			return true
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	return false
+}
+
 // Test key translation with multiple nodes.
 func TestTranslation_Coordinator(t *testing.T) {
 	// Ensure that field key translations requests sent to

From a727c74d353aa1bf4ac025ebb9027500a849d139 Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Tue, 21 Jul 2020 16:52:37 -0500
Subject: [PATCH 02/14] Use nodes from topology to calculate partitionNodes

---
 cluster.go         | 26 ++++++++++++++++++++------
 translator_test.go |  3 +--
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/cluster.go b/cluster.go
index 479a5c6f2..6ef1dc09d 100644
--- a/cluster.go
+++ b/cluster.go
@@ -1027,20 +1027,34 @@ func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool {
 func (c *cluster) partitionNodes(partitionID int) []*Node {
 	// Default replica count to between one and the number of nodes.
 	// The replica count can be zero if there are no nodes.
+
+	// Assume that c.nodes may be missing a node that is part of the cluster but not currently present.
+	// The partition calculation must use the full cluster size in BOTH cases:
+	// - use len(c.Topology.nodeIDs) instead of len(c.nodes),
+	// - collect nodes from c.Topology.nodeIDs rather than from c.nodes,
+	// - when the node is missing, it should be considered, found absent from c.nodes, then omitted from the return slice.
+
 	replicaN := c.ReplicaN
-	if replicaN > len(c.nodes) {
-		replicaN = len(c.nodes)
+	nodeN := len(c.Topology.nodeIDs)
+	if replicaN > nodeN {
+		replicaN = nodeN
 	} else if replicaN == 0 {
 		replicaN = 1
 	}
 
 	// Determine primary owner node.
-	nodeIndex := c.Hasher.Hash(uint64(partitionID), len(c.nodes))
+	nodeIndex := c.Hasher.Hash(uint64(partitionID), nodeN)
 
 	// Collect nodes around the ring.
-	nodes := make([]*Node, replicaN)
+	nodes := make([]*Node, 0, replicaN)
 	for i := 0; i < replicaN; i++ {
-		nodes[i] = c.nodes[(nodeIndex+i)%len(c.nodes)]
+		maybeNodeID := c.Topology.nodeIDs[(nodeIndex+i)%nodeN]
+		for _, node := range c.nodes {
+			if node.ID == maybeNodeID {
+				nodes = append(nodes, node)
+				break
+			}
+		}
 	}
 
 	return nodes
@@ -2193,7 +2207,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
 func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.logger.Printf("merge cluster status: node=%s cluster=%v", c.Node.ID, cs)
+	c.logger.Printf("merge cluster status: node=%s cluster=%v, topologySize=%v", c.Node.ID, cs, len(c.Topology.nodeIDs))
 	// Ignore status updates from self (coordinator).
 	if c.unprotectedIsCoordinator() {
 		return nil
diff --git a/translator_test.go b/translator_test.go
index c78849357..8808add64 100644
--- a/translator_test.go
+++ b/translator_test.go
@@ -297,6 +297,7 @@ func TestTranslation_Reset(t *testing.T) {
 	})
 }
 
+// Test index key translation replication under node failure.
 func TestTranslation_Replication(t *testing.T) {
 	t.Run("Replication", func(t *testing.T) {
 		c := test.MustRunCluster(t, 3,
@@ -325,7 +326,6 @@ func TestTranslation_Replication(t *testing.T) {
 
 		node0 := c[0]
 		node1 := c[1]
-		//node2 := c[2]
 
 		ctx := context.Background()
 		idx := "i"
@@ -352,7 +352,6 @@ func TestTranslation_Replication(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		//exp := `{"results":[{"attrs":{},"columns":[],"keys":["x8","x9","x1","x2","x3","x4","x5","x6","x7"]}]}`
 		exp := `{"results":[{"attrs":{},"columns":[],"keys":["x1","x2"]}]}`
 
 		if !checkClusterState(node0, pilosa.ClusterStateNormal, 1000) {

From 339b76a091bfca98402418f424aca59f9621dcc5 Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Wed, 22 Jul 2020 12:41:58 -0500
Subject: [PATCH 03/14] use c.Topology, when available, to determine
 partitionNodes

---
 cluster.go | 36 +++++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/cluster.go b/cluster.go
index 6ef1dc09d..ed1d92eb9 100644
--- a/cluster.go
+++ b/cluster.go
@@ -110,6 +110,17 @@ func (a Nodes) ContainsID(id string) bool {
 	return false
 }
 
+// NodeByID returns the node for an ID. If the ID is not found,
+// it returns nil.
+func (a Nodes) NodeByID(id string) *Node {
+	for _, n := range a {
+		if n.ID == id {
+			return n
+		}
+	}
+	return nil
+}
+
 // Filter returns a new list of nodes with node removed.
 func (a Nodes) Filter(n *Node) []*Node {
 	other := make([]*Node, 0, len(a))
@@ -1034,8 +1045,22 @@ func (c *cluster) partitionNodes(partitionID int) []*Node {
 	// - collect nodes from c.Topology.nodeIDs rather than from c.nodes,
 	// - when the node is missing, it should be considered, found absent from c.nodes, then omitted from the return slice.
 
+	// Use c.Topology to determine cluster membership when it
+	// exists and contains data. Otherwise, fall back to using
+	// c.nodes. The only time c.Topology should be nil is in
+	// tests.
+	var useTopology bool
+	if c.Topology != nil && len(c.Topology.nodeIDs) > 0 {
+		useTopology = true
+	}
+
 	replicaN := c.ReplicaN
-	nodeN := len(c.Topology.nodeIDs)
+	var nodeN int
+	if useTopology {
+		nodeN = len(c.Topology.nodeIDs)
+	} else {
+		nodeN = len(c.nodes)
+	}
 	if replicaN > nodeN {
 		replicaN = nodeN
 	} else if replicaN == 0 {
@@ -1048,12 +1073,13 @@ func (c *cluster) partitionNodes(partitionID int) []*Node {
 	// Collect nodes around the ring.
 	nodes := make([]*Node, 0, replicaN)
 	for i := 0; i < replicaN; i++ {
-		maybeNodeID := c.Topology.nodeIDs[(nodeIndex+i)%nodeN]
-		for _, node := range c.nodes {
-			if node.ID == maybeNodeID {
+		if useTopology {
+			maybeNodeID := c.Topology.nodeIDs[(nodeIndex+i)%nodeN]
+			if node := Nodes(c.nodes).NodeByID(maybeNodeID); node != nil {
 				nodes = append(nodes, node)
-				break
 			}
+		} else {
+			nodes = append(nodes, c.nodes[(nodeIndex+i)%len(c.nodes)])
 		}
 	}
 

From 5ffc7d7b59b97ba1d09a1807de6ab952dc271adb Mon Sep 17 00:00:00 2001
From: Alan Bernstein <alan.aaron.bernstein@gmail.com>
Date: Wed, 22 Jul 2020 19:54:44 -0500
Subject: [PATCH 04/14] Move checkClusterStatus to test package

---
 server/cluster_test.go | 54 ++++++++++++++++--------------------------
 test/cluster.go        | 12 ++++++++++
 translator_test.go     | 17 ++-----------
 3 files changed, 35 insertions(+), 48 deletions(-)

diff --git a/server/cluster_test.go b/server/cluster_test.go
index d788de803..744b63dcb 100644
--- a/server/cluster_test.go
+++ b/server/cluster_test.go
@@ -143,9 +143,9 @@ func TestClusterResize_AddNode(t *testing.T) {
 		clus := test.MustRunCluster(t, 2)
 		defer clus.Close()
 
-		if !checkClusterState(clus[0], pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(clus[0], pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State())
-		} else if !checkClusterState(clus[1], pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(clus[1], pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State())
 		}
 	})
@@ -176,9 +176,9 @@ func TestClusterResize_AddNode(t *testing.T) {
 		}
 		defer m1.Close()
 
-		if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
-		} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
 		}
 	})
@@ -224,9 +224,9 @@ func TestClusterResize_AddNode(t *testing.T) {
 		}
 		defer m1.Close()
 
-		if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
-		} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
 		}
 
@@ -273,9 +273,9 @@ func TestClusterResize_AddNode(t *testing.T) {
 		}
 		defer m1.Close()
 
-		if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
-		} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
 		}
 
@@ -326,9 +326,9 @@ func TestClusterResize_AddNode(t *testing.T) {
 		}
 		defer m1.Close()
 
-		if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
-		} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
 		}
 
@@ -373,9 +373,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		}
 		defer m1.Close()
 
-		if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
-		} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
 		}
 
@@ -431,9 +431,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		}()
 		defer m1.Close()
 
-		if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
-		} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
 		}
 
@@ -489,9 +489,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		}
 		defer m1.Close()
 
-		if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
-		} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
 		}
 
@@ -545,9 +545,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		}
 		defer m1.Close()
 
-		if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
-		} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
 		}
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
@@ -598,11 +598,11 @@ func TestCluster_GossipMembership(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
-		} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
-		} else if !checkClusterState(m2, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(m2, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node2 cluster state: %s", m2.API.State())
 		}
 
@@ -725,15 +725,3 @@ func TestClusterMutualTLS(t *testing.T) {
 		t.Fatal(err)
 	}
 }
-
-// checkClusterState polls a given cluster for its state until it
-// receives a matching state. It polls up to n times before returning.
-func checkClusterState(m *test.Command, state string, n int) bool {
-	for i := 0; i < n; i++ {
-		if m.API.State() == state {
-			return true
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	return false
-}
diff --git a/test/cluster.go b/test/cluster.go
index babb3890f..4fbc2050e 100644
--- a/test/cluster.go
+++ b/test/cluster.go
@@ -193,6 +193,18 @@ func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu
 	return c
 }
 
+// CheckClusterState polls a given cluster for its state until it
+// receives a matching state. It polls up to n times before returning.
+func CheckClusterState(m *Command, state string, n int) bool {
+	for i := 0; i < n; i++ {
+		if m.API.State() == state {
+			return true
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	return false
+}
+
 // newCluster creates a new cluster
 func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluster, error) {
 	if size == 0 {
diff --git a/translator_test.go b/translator_test.go
index 8808add64..0acce9b66 100644
--- a/translator_test.go
+++ b/translator_test.go
@@ -22,7 +22,6 @@ import (
 	"io"
 	"reflect"
 	"testing"
-	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/pilosa/pilosa/v2"
@@ -354,9 +353,9 @@ func TestTranslation_Replication(t *testing.T) {
 
 		exp := `{"results":[{"attrs":{},"columns":[],"keys":["x1","x2"]}]}`
 
-		if !checkClusterState(node0, pilosa.ClusterStateNormal, 1000) {
+		if !test.CheckClusterState(node0, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", node0.API.State())
-		} else if !checkClusterState(node1, pilosa.ClusterStateNormal, 1000) {
+		} else if !test.CheckClusterState(node1, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", node1.API.State())
 		}
 
@@ -373,18 +372,6 @@ func TestTranslation_Replication(t *testing.T) {
 	})
 }
 
-// checkClusterState polls a given cluster for its state until it
-// receives a matching state. It polls up to n times before returning.
-func checkClusterState(m *test.Command, state string, n int) bool {
-	for i := 0; i < n; i++ {
-		if m.API.State() == state {
-			return true
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	return false
-}
-
 // Test key translation with multiple nodes.
 func TestTranslation_Coordinator(t *testing.T) {
 	// Ensure that field key translations requests sent to

From 75930ed82fa5605c193005bfaefaeee603b8037d Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Thu, 23 Jul 2020 09:37:50 -0600
Subject: [PATCH 05/14] Remove generation of rbf/fun.dot in tests

---
 rbf/cursor_test.go | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go
index 9e98fafb3..0db356177 100644
--- a/rbf/cursor_test.go
+++ b/rbf/cursor_test.go
@@ -1035,7 +1035,6 @@ func TestCursor_PlayContainer(t *testing.T) {
 	if err := cur.First(); err != nil {
 		panic(err)
 	}
-	cur.Dump("fun.dot")
 }
 
 func TestCursor_OneBitmap(t *testing.T) {
@@ -1071,7 +1070,6 @@ func TestCursor_OneBitmap(t *testing.T) {
 	if err := cur.First(); err != nil {
 		panic(err)
 	}
-	cur.Dump("fun.dot")
 }
 func TestCursor_GenerateAll(t *testing.T) {
 	db := MustOpenDB(t)
@@ -1111,9 +1109,4 @@ func TestCursor_GenerateAll(t *testing.T) {
 	if _, err := tx.AddRoaring("field/view/", bb); err != nil {
 		panic(err)
 	}
-	cur, err := tx.Cursor("field/view/")
-	if err != nil {
-		panic(err)
-	}
-	cur.Dump("fun.dot")
 }

From ac7be132ef5da5e342f3f157fdb8048bdbc4a169 Mon Sep 17 00:00:00 2001
From: Jason Aten <jason.aten@molecula.com>
Date: Tue, 21 Jul 2020 11:36:09 -0400
Subject: [PATCH 06/14] Tx integration milestone

a) All tests green under -race for both PILOSA_TXSRC=roaring and PILOSA_TXSRC=badger.

b) Distinct is merged back into mainline pilosa.

Seebs notes on the Distinct work:

merge Distinct plugin back into main source tree, convert to Tx

We drop all references to the Preemptively Deprecated Don't You Dare
Use This extension interface, and move the one and only extension we had
(Distinct) into the main executor.

Also this fixes an arguable bug, which is that Container.AsBitmap()
would panic on a nil parameter, but it should have returned an empty
bitmap, because a nil *Ccontainer is a valid empty container. This
simplifies logic significantly in Distinct.

Fixes #569 #570 #571 #572 #573 #584 #585
---
 Makefile                   |   60 +-
 api.go                     |    9 +-
 badger.go                  |  418 ++++++++++----
 badger_test.go             |  263 ++++++---
 bluegreentx.go             |  284 ++++++++--
 catcher.go                 |   28 +-
 cluster.go                 |    2 +-
 cluster_internal_test.go   |   13 +-
 executor.go                |  282 +++++-----
 executor_test.go           |   34 +-
 extension.go               |   96 ----
 field.go                   |   20 +-
 field_internal_test.go     |   58 +-
 field_test.go              |   40 +-
 fragment.go                |  169 ++++--
 fragment_internal_test.go  | 1055 ++++++++++++++++++++++++++++++++----
 generation.go              |   23 +
 generation_test.go         |    7 +-
 holder.go                  |   48 +-
 holder_test.go             |   22 +-
 index.go                   |   63 ++-
 index_internal_test.go     |    9 +-
 mmap_test.go               |    4 +-
 pql/ast.go                 |   34 +-
 rbf/db.go                  |    4 +
 roaring/container_stash.go |    6 +-
 row.go                     |  118 ----
 server.go                  |   86 +--
 test/index.go              |   12 +-
 tx.go                      |  213 +++++++-
 txfactory.go               |  187 +++++--
 utils_internal_test.go     |   41 +-
 view.go                    |  104 ++--
 view_internal_test.go      |   10 +-
 vprint.go                  |   30 +
 35 files changed, 2795 insertions(+), 1057 deletions(-)
 delete mode 100644 extension.go

diff --git a/Makefile b/Makefile
index e1577b62b..b8454402f 100644
--- a/Makefile
+++ b/Makefile
@@ -16,16 +16,13 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE))
 NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
 BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
 BUILD_TAGS += shardwidth$(SHARD_WIDTH)
-BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p))
 define LICENSE_HASH_CODE
     head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
 endef
 LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
 
-PLUGINS=distinct
 export GO111MODULE=on
 export GOPRIVATE=github.com/molecula
-export PLUGINS
 
 # Run tests and compile Pilosa
 default: test build
@@ -97,7 +94,7 @@ clustertests: vendor
 
 # Like clustertests, but rebuilds all images.
 clustertests-build: vendor
-	docker-compose -f $(DOCKER_COMPOSE) down
+	docker-compose -f $(DOCKER_COMPOSE) down -v
 	docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
 
 # Create prerelease builds
@@ -152,25 +149,80 @@ docker-test:
 # run top tests, not subdirs. print summary red/green after.
 # The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
 topt:
+	mv log.topt.roar log.topt.roar.prev || true
 	go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.roar
 	@echo "   log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
 	@echo "   log.topt.roar   red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l
 
 topt-badger:
+	mv log.topt.badger log.topt.badger.prev || true
 	PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger
 	@echo "   log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l
 	@echo "   log.topt.badger   red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
 
+topt-rb:
+	mv log.topt.roaring_badger log.topt.roaring_badger.prev || true
+	PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger
+	@echo "   log.topt.roaring_badger green: \c"; cat log.topt.roaring_badger | grep PASS |wc -l
+	@echo "   log.topt.roaring_badger   red: \c"; cat log.topt.roaring_badger | grep '\-\-\- FAIL' |wc -l
+
+topt-badger-race:
+	mv log.topt.badger-race log.topt.badger-race.prev || true
+	PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger-race
+	@echo "   log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l
+	@echo "   log.topt.badger-race   red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l
+
 topt-rbf:
+	mv log.topt.rbf log.topt.rbf.prev || true
 	PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.rbf
 	@echo "   log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l
 	@echo "   log.topt.rbf   red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l
 
 topt-race:
+	mv log.topt.race log.topt.race.prev || true
 	go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.race
 	@echo "   log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
 	@echo "   log.topt.race   red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l
 
+# blue-green checks. These run two different storage engines (rbf, roaring, or badger)
+# and compare each transaction for a result.
+bg-br:
+	mv log.bg.bg_roar log.bg.bg_roar.prev || true
+	PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.bg_roar
+	@echo "   log.bg.bg_roar green: \c"; cat log.bg.bg_roar | grep PASS |wc -l
+	@echo "   log.bg.bg_roar   red: \c"; cat log.bg.bg_roar | grep '\-\-\- FAIL' |wc -l
+
+bg-rb:
+	mv log.bg.roar_bg log.bg.roar_bg.prev || true
+	PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.roar_bg
+	@echo "   log.bg.roar_bg green: \c"; cat log.bg.roar_bg | grep PASS |wc -l
+	@echo "   log.bg.roar_bg   red: \c"; cat log.bg.roar_bg | grep '\-\-\- FAIL' |wc -l
+
+bg-fr:
+	mv log.bg.rbf_roar log.bg.rbf_roar.prev || true
+	PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.rbf_roar
+	@echo "   log.bg.rbf_roar green: \c"; cat log.bg.rbf_roar | grep PASS |wc -l
+	@echo "   log.bg.rbf_roar   red: \c"; cat log.bg.rbf_roar | grep '\-\-\- FAIL' |wc -l
+
+bg-rf:
+	mv log.bg.roar_rbf log.bg.roar_rbf.prev || true
+	PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.roar_rbf
+	@echo "   log.bg.roar_rbf green: \c"; cat log.bg.roar_rbf | grep PASS |wc -l
+	@echo "   log.bg.roar_rbf   red: \c"; cat log.bg.roar_rbf | grep '\-\-\- FAIL' |wc -l
+
+bg-fb:
+	mv log.bg.rbf_badger log.bg.rbf_badger.prev || true
+	PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.rbf_badger
+	@echo "   log.bg.rbf_badger green: \c"; cat log.bg.rbf_badger | grep PASS |wc -l
+	@echo "   log.bg.rbf_badger   red: \c"; cat log.bg.rbf_badger | grep '\-\-\- FAIL' |wc -l
+
+bg-bf:
+	mv log.bg.badger_rbf log.bg.badger_rbf.prev || true
+	PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.badger_rbf
+	@echo "   log.bg.badger_rbf green: \c"; cat log.bg.badger_rbf | grep PASS |wc -l
+	@echo "   log.bg.badger_rbf   red: \c"; cat log.bg.badger_rbf | grep '\-\-\- FAIL' |wc -l
+
+
 # Run golangci-lint
 golangci-lint: require-golangci-lint
 	golangci-lint run --skip-files '.*\.peg\.go'
diff --git a/api.go b/api.go
index b5bc89422..3ed7358e3 100644
--- a/api.go
+++ b/api.go
@@ -371,16 +371,9 @@ func importWorker(importWork chan importJob) {
 				var doClear bool
 				switch doAction {
 				case RequestActionOverwrite:
-					// TODO(jea): the question here is, why are we commiting this separately from j.tx?
-					// why doesn't j.tx suffice? It doesn't but why/which is correct?
-					tx := j.field.holder.indexes[j.field.index].Txf.NewTx(Txo{Write: true, Field: j.field})
-					defer tx.Rollback()
-					if err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block); err != nil {
+					if err := j.field.importRoaringOverwrite(j.ctx, j.tx, viewData, j.shard, viewName, j.req.Block); err != nil {
 						return errors.Wrap(err, "importing roaring as overwrite")
 					}
-					if err := tx.Commit(); err != nil {
-						return errors.Wrap(err, "commit of importing roaring as overwrite")
-					}
 				case RequestActionClear:
 					doClear = true
 					fallthrough
diff --git a/badger.go b/badger.go
index 65052b525..aee8d9897 100644
--- a/badger.go
+++ b/badger.go
@@ -15,8 +15,9 @@
 package pilosa
 
 import (
-	"errors"
+	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -29,7 +30,9 @@ import (
 	"unsafe"
 
 	badger "github.com/dgraph-io/badger/v2"
+	badgeroptions "github.com/dgraph-io/badger/v2/options"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pkg/errors"
 )
 
 // TODO: is there a more optimal time to do badger garbage collection?
@@ -189,15 +192,70 @@ func (l *BadgerLog) Debugf(f string, v ...interface{}) {
 	l.Printf("DEBUG: "+f, v...)
 }
 
+// badgerRegistrar facilitates shutdown
+// of all the badger databases started under
+// tests. Its needed because most tests don't cleanup
+// the *Index(es) they create. But we still
+// want to shutdown badgerDB goroutines
+// after tests run.
+//
+// It also allows opening the same path twice to
+// result in sharing the same open database handle, and
+// thus the same transactional guarantees.
+//
+type badgerRegistrar struct {
+	mu sync.Mutex
+	mp map[*BadgerDBWrapper]bool
+
+	path2db map[string]*BadgerDBWrapper
+}
+
+var globalBadgerReg *badgerRegistrar = newBadgerTestRegistrar()
+
+func newBadgerTestRegistrar() *badgerRegistrar {
+	return &badgerRegistrar{
+		mp:      make(map[*BadgerDBWrapper]bool),
+		path2db: make(map[string]*BadgerDBWrapper),
+	}
+}
+
+// register each badger created under tests, so we
+// can clean them up. This is called by openBadgerDBWrapper() while
+// holding the r.mu.Lock, since it needs to atomically
+// check the registry and make a new instance only
+// if one does not exist for its path, and otherwise
+// return the existing instance.
+func (r *badgerRegistrar) unprotectedRegister(w *BadgerDBWrapper) {
+	r.mp[w] = true
+	r.path2db[w.path] = w
+}
+
+// unregister removes w from r
+func (r *badgerRegistrar) unregister(w *BadgerDBWrapper) {
+	r.mu.Lock()
+	delete(r.mp, w)
+	delete(r.path2db, w.path)
+	r.mu.Unlock()
+}
+
+func DumpAllBadger() {
+	globalBadgerReg.mu.Lock()
+	defer globalBadgerReg.mu.Unlock()
+	for w := range globalBadgerReg.mp {
+		_ = w
+		AlwaysPrintf("this badger path='%v' has: \n%v\n", w.path, w.StringifiedBadgerKeys(nil))
+	}
+}
+
 // newBadgerDBWrapper creates a new empty database, blowing away
 // any prior path + "-badgerdb" directory.
-func newBadgerDBWrapper(path string) (*BadgerDBWrapper, error) {
+func (r *badgerRegistrar) newBadgerDBWrapper(path string) (*BadgerDBWrapper, error) {
 	bpath := badgerPath(path)
 	err := os.RemoveAll(bpath)
 	if err != nil {
 		return nil, err
 	}
-	return openBadgerDBWrapper(bpath)
+	return r.openBadgerDBWrapper(bpath)
 }
 
 // badgerPath is a helper for determining the full directory
@@ -212,7 +270,12 @@ func badgerPath(path string) string {
 // openBadgerDB opens the database in the bpath directoy
 // without deleting any prior content. Any BadgerDB
 // database directory will have the "-badgerdb" suffix.
-func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
+//
+// openBadgerDB will check the registry and make a new instance only
+// if one does not exist for its bpath. Otherwise it returns
+// the existing instance. This insures only one badgerDB
+// per bpath in this pilosa node.
+func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
 
 	// now that newTxFactory can call us directly, we might not
 	// have the -badgerdb suffix.
@@ -220,9 +283,33 @@ func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
 		bpath += "-badgerdb"
 	}
 
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	w, ok := r.path2db[bpath]
+	if ok {
+		// creates the effect of having only one badger open per pilosa node.
+		return w, nil
+	}
+	// otherwise, make a new badger and store it in globalBadgerReg
+
 	// regular: works on amd64, but 386 doesn't work.
 	opt := badger.DefaultOptions(bpath).WithLogger(badgerDefaultLogger)
 
+	opt.Compression = badgeroptions.None // turn off compression.
+	opt.ZSTDCompressionLevel = 0         // really, just in case.
+
+	// MaxCacheSize docs:
+	//
+	// how much data cache should hold in memory. A small size of
+	// cache means lower memory consumption and lookups/iterations
+	// would take longer. It is recommended to use a cache if you're
+	// using compression or encryption. If compression and
+	// encryption both are disabled, adding a cache will lead to
+	// unnecessary overhead which will affect the read performance.
+	// Setting size to zero disables the cache altogether.
+	opt.MaxCacheSize = 0
+	opt.LoadBloomsOnOpen = false // should speed up start-up time.
+
 	// to get memory only do:
 	//opt := badger.DefaultOptions("").WithLogger(badgerDefaultLogger).WithInMemory(true)
 
@@ -231,12 +318,16 @@ func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
 		return nil, err
 	}
 	halt := make(chan bool)
-	w := &BadgerDBWrapper{
+	w = &BadgerDBWrapper{
+		reg:    r,
 		path:   bpath,
 		db:     db,
 		halt:   halt,
 		hasher: NewBlake3Hasher(),
 	}
+	r.unprotectedRegister(w)
+
+	w.startStack = stack()
 	w.startBadgerGarbageCollectionBackgroundGoro()
 	return w, nil
 }
@@ -251,77 +342,8 @@ func (w *BadgerDBWrapper) DeleteIndex(indexName string) error {
 	if strings.Contains(indexName, "'") {
 		return fmt.Errorf("error: bad indexName `%v` in BadgerDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName)
 	}
-	w.muDb.Lock()
-	defer w.muDb.Unlock()
-
-	// a) do key-ony iteration, no value fetch;
-	//
-	// b) do deletes in large batches, to avoid alot of txn overhead;
-	//    per recommendation https://github.com/dgraph-io/badger/issues/598
-	//
-	// c) we do not, at present, try to maintain one large
-	//    transaction with all the keys in a index in it. Because
-	//    there can be too many keys. Hence the index will disappear
-	//    in chucks of 100K keys, not atomically-all-at-once.
-
 	prefix := badgerIndexOnlyPrefix(indexName)
-
-	noMoreKeysWithPrefix := false
-	const maxDeletesPerTxn = 100000
-
-	for !noMoreKeysWithPrefix {
-		err := w.db.Update(func(txn *badger.Txn) error {
-			o := badger.DefaultIteratorOptions
-			o.AllVersions = false
-			o.PrefetchValues = false // key-only iteration, no values.
-
-			// note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here?
-			// This means we messed up and Closed() the Database already; too early.
-			it := txn.NewIterator(o)
-
-			defer it.Close()
-			n := 0
-			goners := make([][]byte, 0, maxDeletesPerTxn)
-			for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
-
-				// KeyCopy() is required; Key() means corruption and possible segfault.
-				key := it.Item().KeyCopy(nil)
-				goners = append(goners, key)
-				n++
-				if n >= maxDeletesPerTxn {
-					break
-				}
-			}
-			if !it.ValidForPrefix(prefix) {
-				noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn
-			}
-			for _, key := range goners {
-				if err := txn.Delete(key); err != nil {
-					return err
-				}
-			}
-			return nil // auto-commit happens
-		})
-		// err back from Update can be ErrConflict in case of
-		// a conflict. Badger docs: "Depending on the state
-		// of your application, you have the option to
-		// retry the operation if you receive this error."
-		panicOn(err)
-
-	} // end for: proceed to next bath of 100K keys
-
-	// Finally, run a garbage collection to delete values from the value log.
-	//
-	// "Only one GC is allowed at a time. If another value log GC
-	// is running, or DB has been closed, this would return an ErrRejected."
-	//   -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC
-	// Still, we don't see a mutex inside the RunValueLogGC code, so
-	// lock muGC just to be sure.
-	w.muGC.Lock()
-	defer w.muGC.Unlock()
-	_ = w.db.RunValueLogGC(0.5)
-
-	return nil
+	return w.DeletePrefix(prefix)
 }
 
 // startBadgerGarbageCollectionBackgroundGoro handles Badger DB
@@ -367,6 +389,9 @@ type BadgerDBWrapper struct {
 	path string
 	db   *badger.DB
 
+	// track our registrar for Close / goro leak reporting purposes.
+	reg *badgerRegistrar
+
 	// openTx and openIt are BadgerDBWrapper scoped tables of all open
 	// transactions and iterators. These are primarily for debugging purposes.
 	// openTx and openIt should only be read/written after locking the muOpenTxIt mutex.
@@ -404,6 +429,10 @@ type BadgerDBWrapper struct {
 	// safety because otherwise TestAPI_ImportColumnAttrs sees
 	// corrupted data.
 	doAllocZero bool
+
+	// stack() from our creation point, to track tests
+	// that haven't closed us.
+	startStack string
 }
 
 // unprotectedListOpenTxAsString is a debugging helper.
@@ -437,24 +466,30 @@ func (w *BadgerDBWrapper) UnprotectedListOpenItAsString() (r string) {
 // Read-only queries should set write to false, to allow more concurrency.
 // Methods on a BadgerTx are thread-safe, and can be called from
 // different goroutines.
-func (w *BadgerDBWrapper) NewBadgerTx(write bool) (tx *BadgerTx) {
+//
+// initialIndexName is optional. It is set by the TxFactory from the Txo
+// options provided at the Tx creation point. It allows us to recognize
+// and isolate cross-index queries more quickly. It can always be empty ""
+// but when set is highly useful for debugging. It has no impact
+// on transaction behavior.
+//
+func (w *BadgerDBWrapper) NewBadgerTx(write bool, initialIndexName string) (tx *BadgerTx) {
 	w.muDb.Lock()
 	defer w.muDb.Unlock()
 
 	tx = &BadgerTx{
-		write:       write,
-		tx:          w.db.NewTransaction(write),
-		Db:          w,
-		initloc:     stack(),
-		doAllocZero: w.doAllocZero,
+		write:            write,
+		tx:               w.db.NewTransaction(write),
+		Db:               w,
+		initloc:          stack(),
+		doAllocZero:      w.doAllocZero,
+		initialIndexName: initialIndexName,
 	}
-	//vv("NewBadgerTx(write=%v) top, p=%p", write, tx)
-	//pp("NewBadgerTx(write=%v) top, p=%p, stack=\n\n'%v'", write, tx, stack())
 
 	if w.openTx == nil {
 		w.openTx = make(map[*BadgerTx]bool)
 	}
-	//pp("NewBadgerTx(write=%v); p=%p; (currently open txn: '%v', its: '%v'). initloc:'%v'", write, tx, w.unprotectedListOpenTxAsString(), w.UnprotectedListOpenItAsString(), tx.initloc)
+
 	w.muOpenTxIt.Lock()
 	w.openTx[tx] = write
 	w.muOpenTxIt.Unlock()
@@ -466,6 +501,7 @@ func (w *BadgerDBWrapper) Close() (err error) {
 	w.muDb.Lock()
 	defer w.muDb.Unlock()
 	if !w.closed {
+		w.reg.unregister(w)
 		close(w.halt)
 		w.closed = true
 	}
@@ -501,8 +537,15 @@ type BadgerTx struct {
 	// for tracking txn boundary issues, track all the memory
 	// that we deploy for roaring containers, and zero it on
 	// transaction commit/rollback.
+	acMu          sync.Mutex // protect ourAllocs and ourContainers
 	ourAllocs     [][]byte
 	ourContainers []*roaring.Container
+
+	initialIndexName string
+}
+
+func (tx *BadgerTx) Type() string {
+	return BadgerTxn
 }
 
 func (tx *BadgerTx) UseRowCache() bool {
@@ -521,6 +564,8 @@ func (tx *BadgerTx) UseRowCache() bool {
 // to transaction commit.
 func (tx *BadgerTx) overWriteOurAllocs() {
 
+	tx.acMu.Lock()
+	defer tx.acMu.Unlock()
 	for _, s := range tx.ourAllocs {
 
 		// The Go compiler recognizes the following pattern and inserts
@@ -529,6 +574,10 @@ func (tx *BadgerTx) overWriteOurAllocs() {
 		// and https://codereview.appspot.com/137880043
 		for i := range s {
 			s[i] = 0
+			// or
+			// Seebs suggested we might see even more crashes :)
+			// but since it will be slow (no memclr), we'll leave the default 0 for now.
+			//s[i] = -2
 		}
 	}
 	// keep this around if we need to activate out-of-mmap memory access again.
@@ -632,21 +681,57 @@ func badgerKey(index, field, view string, shard uint64, roaringContainerKey uint
 
 	prefix := badgerPrefix(index, field, view, shard)
 	ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
-	return append(prefix, ckey...)
+	bkey := append(prefix, ckey...)
+	MustValidateKey(bkey)
+	return bkey
+}
+
+var ckeyPartExpected = []byte(";ckey@")
+
+// MustValidatekey will panic on a bad badgerKey with an informative message.
+func MustValidateKey(bkey []byte) {
+	n := len(bkey)
+	if n < 56 {
+		panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey)))
+	}
+	beforeCkey := bkey[n-26 : n-20]
+	if !bytes.Equal(beforeCkey, ckeyPartExpected) {
+		panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey)))
+	}
+}
+
+func shardFromBadgerKey(bkey []byte) (shard uint64) {
+	MustValidateKey(bkey)
+
+	n := len(bkey)
+	// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 -> idx:'i';fld:'f';vw:'standard';shd:'1
+	by := bkey[:n-27]
+	beg := bytes.LastIndex(by, []byte("'"))
+	if beg == -1 {
+		panic(fmt.Sprintf("bad bkey='%v' did not have single quote to being shard decoding", string(bkey)))
+	}
+	parseMe := string(by[beg+1:])
+	shard, err := strconv.ParseUint(parseMe, 10, 64)
+	if err != nil {
+		panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err))
+	}
+	return shard
 }
 
 // badgerKeyAndPrefix returns the equivalent of badgerKey() and badgerPrefix() calls.
 func badgerKeyAndPrefix(index, field, view string, shard uint64, roaringContainerKey uint64) (key, prefix []byte) {
 	prefix = badgerPrefix(index, field, view, shard)
 	ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
-	return append(prefix, ckey...), prefix
+	bkey := append(prefix, ckey...)
+	MustValidateKey(bkey)
+	return bkey, prefix
 }
 
 var _ = badgerKeyAndPrefix // keep linter happy
 
 // badgerKeyExtractContainerKey extracts the containerKey from bkey.
 func badgerKeyExtractContainerKey(bkey []byte) (containerKey uint64) {
-
+	MustValidateKey(bkey)
 	// The zero padding means that the container-key is always the last 20 bytes of the bkey.
 	//
 	// Be sure to catch the problematic case of a user passing in only a prefix. A prefix
@@ -665,11 +750,15 @@ func badgerKeyExtractContainerKey(bkey []byte) (containerKey uint64) {
 	return
 }
 
+func badgerAllShardPrefix(index, field, view string) []byte {
+	return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:", index, field, view))
+}
+
 // badgerPrefix returns everything from badgerKey up to and
 // including the '@' fune in a badger key. The prefix excludes the roaring container key itself.
 // NB must be kept in sync with badgerKey() and badgerKeyExtractContainerKey().
 func badgerPrefix(index, field, view string, shard uint64) []byte {
-	return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%x';ckey@", index, field, view, shard))
+	return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard))
 }
 
 // badgerIndexOnlyPrefix returns a prefix suitable for DeleteIndex and a key-scan to
@@ -869,6 +958,38 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64
 	return exists, err
 }
 
+func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
+
+	prefix := badgerAllShardPrefix(index, field, view)
+
+	bi := NewBadgerIterator(tx, prefix)
+	defer bi.Close()
+	bi.Seek(prefix)
+	if !bi.it.Valid() {
+		return
+	}
+	lastShard := uint64(0)
+	firstDone := false
+	for bi.Next() {
+		item := bi.it.Item()
+		key := item.Key()
+		shard := shardFromBadgerKey(key)
+		if firstDone {
+			if shard != lastShard {
+				sliceOfShards = append(sliceOfShards, shard)
+			}
+			lastShard = shard
+		} else {
+			// first time
+			lastShard = shard
+			firstDone = true
+			sliceOfShards = append(sliceOfShards, shard)
+		}
+
+	}
+	return
+}
+
 // key is the container key for the first roaring Container
 // roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will
 // return the first container at or after key. found will be true if a
@@ -877,10 +998,10 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64
 // BadgerTx notes: We auto-stop at the end of this shard, not going beyond.
 func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
 
-	// needle example: "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000"
+	// needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000"
 	needle := badgerKey(index, field, view, shard, firstRoaringContainerKey)
 
-	// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
+	// prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@"
 	prefix := badgerPrefix(index, field, view, shard)
 
 	bi := NewBadgerIterator(tx, prefix)
@@ -917,12 +1038,6 @@ func NewBadgerIterator(tx *BadgerTx, prefix []byte) (bi *BadgerIterator) {
 	tx.Db.muOpenTxIt.Lock()
 	defer tx.Db.muOpenTxIt.Unlock()
 
-	defer func() {
-		r := recover()
-		if r != nil {
-			panic(r)
-		}
-	}()
 	opts := badger.DefaultIteratorOptions
 	opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow.
 	opts.Reverse = false
@@ -1125,6 +1240,7 @@ func (tx *BadgerTx) Count(index, field, view string, shard uint64) (uint64, erro
 }
 
 // Max is the maximum bit-value in your bitmap.
+// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does.
 func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error) {
 
 	prefix := badgerPrefix(index, field, view, shard)
@@ -1133,7 +1249,10 @@ func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error)
 	it := NewBadgerReverseIterator(tx, prefix, seekto) // this iterator is still open, when we commit/discard tx.
 	defer it.Close()
 
-	hb, rc := it.Value()
+	if !it.it.Valid() {
+		return 0, nil
+	}
+	hb, rc := it.Value() // getting it returns invalid, as in empty iterator
 	lb := rc.Max()
 
 	return hb<<16 | uint64(lb), nil
@@ -1309,7 +1428,7 @@ func (tx *BadgerTx) IncrementOpN(index, field, view string, shard uint64, change
 
 // ImportRoaringBits handles deletes by setting clear=true.
 // rowSet[rowID] returns the number of bit changed on that rowID.
-func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
+func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
 	n := itr.Len()
 	if n == 0 {
 		return
@@ -1456,11 +1575,19 @@ func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
 	// TODO: performance tuning might want w := v here, if we can guarantee no access to memory past the Tx lifetime.
 	//
 	// Problem is, at least some tests appear to not respect transaction boundaries...
+	//
+	// Seebs suggested this nice variation: we could use individual mmaps for these
+	// copies, which would be unusable in production, but workable for testing, and then unmap them,
+	// which would get us probable segfaults on future accesses to them.
+	//
 	w := make([]byte, len(v))
-	copy(w, v) // green go test -v -run TestAPI_ImportColumnAttrs
+	copy(w, v)
+	// the copy above makes green: // green go test -v -run TestAPI_ImportColumnAttrs
 	//w := v // if instead of append we use v directly, it causes red: go test -v -run TestAPI_ImportColumnAttrs
 
 	// register w so we can catch out-of-tx memory access
+	tx.acMu.Lock()
+	defer tx.acMu.Unlock()
 	tx.ourAllocs = append(tx.ourAllocs, w)
 
 	switch typ {
@@ -1511,7 +1638,7 @@ func fromInterval16(a []roaring.Interval16) []byte {
 // keys available in badger.
 func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx) (r string) {
 	if optionalUseThisTx == nil {
-		tx := w.NewBadgerTx(!writable)
+		tx := w.NewBadgerTx(!writable, "<StringifiedBadgerKeys>")
 		defer tx.Rollback()
 		r = stringifiedBadgerKeysTx(tx)
 		return
@@ -1699,3 +1826,94 @@ func dirAsString(path string) (r string) {
 }
 
 var _ = dirAsString // happy linter
+
+func (w *BadgerDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
+	prefix := badgerPrefix(index, field, view, shard)
+	return w.DeletePrefix(prefix)
+}
+
+func (w *BadgerDBWrapper) DeletePrefix(prefix []byte) error {
+	w.muDb.Lock()
+	defer w.muDb.Unlock()
+
+	// a) do key-ony iteration, no value fetch;
+	//
+	// b) do deletes in large batches, to avoid alot of txn overhead;
+	//    per recommendation https://github.com/dgraph-io/badger/issues/598
+	//
+	// c) we do not, at present, try to maintain one large
+	//    transaction with all the keys in a index in it. Because
+	//    there can be too many keys. Hence the index will disappear
+	//    in chucks of 100K keys, not atomically-all-at-once.
+
+	noMoreKeysWithPrefix := false
+	const maxDeletesPerTxn = 100000
+
+	for !noMoreKeysWithPrefix {
+		err := w.db.Update(func(txn *badger.Txn) error {
+			o := badger.DefaultIteratorOptions
+			o.AllVersions = false
+			o.PrefetchValues = false // key-only iteration, no values.
+
+			// note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here?
+			// This means we messed up and Closed() the Database already; too early.
+			it := txn.NewIterator(o)
+
+			defer it.Close()
+			n := 0
+			goners := make([][]byte, 0, maxDeletesPerTxn)
+			for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
+
+				// KeyCopy() is required; Key() means corruption and possible segfault.
+				key := it.Item().KeyCopy(nil)
+				goners = append(goners, key)
+				n++
+				if n >= maxDeletesPerTxn {
+					break
+				}
+			}
+			if !it.ValidForPrefix(prefix) {
+				noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn
+			}
+			for _, key := range goners {
+				if err := txn.Delete(key); err != nil {
+					return err
+				}
+			}
+			return nil // auto-commit happens
+		})
+		// err back from Update can be ErrConflict in case of
+		// a conflict. Badger docs: "Depending on the state
+		// of your application, you have the option to
+		// retry the operation if you receive this error."
+		panicOn(err)
+
+	} // end for: proceed to next bath of 100K keys
+
+	// Finally, run a garbage collection to delete values from the value log.
+	//
+	// "Only one GC is allowed at a time. If another value log GC
+	// is running, or DB has been closed, this would return an ErrRejected."
+	//   -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC
+	// Still, we don't see a mutex inside the RunValueLogGC code, so
+	// lock muGC just to be sure.
+	w.muGC.Lock()
+	defer w.muGC.Unlock()
+	_ = w.db.RunValueLogGC(0.5)
+
+	return nil
+}
+
+func (tx *BadgerTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+
+	rbm, err := tx.RoaringBitmap(index, field, view, shard)
+	if err != nil {
+		return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap")
+	}
+	var buf bytes.Buffer
+	sz, err = rbm.WriteTo(&buf)
+	if err != nil {
+		return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)")
+	}
+	return ioutil.NopCloser(&buf), sz, err
+}
diff --git a/badger_test.go b/badger_test.go
index 4f2f09f52..10fc0baa0 100644
--- a/badger_test.go
+++ b/badger_test.go
@@ -12,12 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// explanation of build tags:
+//
 // badgerdb builds but won't run in 32-bit 386 world, as of 2020 July 20.
 // See https://github.com/dgraph-io/badger/issues/1384 for any progress.
 // What we see is that the value-log allocations immediately run out of
 // memory. So we turn off 386 with a build tag to keep the .circleci happy.
+//
+// gendebug_test will have a TestMain if build tag generationdebug is on,
+// so we avoid conflicting with that debug scenario.
 
 // +build !386
+// +build !generationdebug
 
 package pilosa
 
@@ -40,7 +46,7 @@ var _ = &roaring.Bitmap{}
 
 func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
 
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, index)
 	defer tx.Rollback()
 	exists, err := tx.Contains(index, field, view, shard, bitvalue)
 	panicOn(err)
@@ -53,7 +59,7 @@ func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string
 
 func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
 
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, index)
 	defer tx.Rollback()
 	exists, err := tx.Contains(index, field, view, shard, bitvalue)
 	panicOn(err)
@@ -64,7 +70,7 @@ func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view str
 }
 
 func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
-	tx := dbwrap.NewBadgerTx(writable)
+	tx := dbwrap.NewBadgerTx(writable, index)
 
 	// add a bit
 	changed, err := tx.Add(index, field, view, shard, doBatched, putme)
@@ -82,14 +88,14 @@ func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string,
 }
 
 func badgerDBMustDeleteBitvalueContainer(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
-	tx := dbwrap.NewBadgerTx(writable)
+	tx := dbwrap.NewBadgerTx(writable, index)
 	hi := highbits(putme)
 	panicOn(tx.RemoveContainer(index, field, view, shard, hi))
 	panicOn(tx.Commit())
 }
 
 func badgerDBMustDeleteBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
-	tx := dbwrap.NewBadgerTx(writable)
+	tx := dbwrap.NewBadgerTx(writable, index)
 	_, err := tx.Remove(index, field, view, shard, putme)
 	panicOn(err)
 	panicOn(tx.Commit())
@@ -99,7 +105,7 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()
 	var err error
 	fn := badgerPath(path)
 	panicOn(os.RemoveAll(fn))
-	w, err = newBadgerDBWrapper(path)
+	w, err = globalBadgerReg.newBadgerDBWrapper(path)
 	panicOn(err)
 
 	// verify it is empty
@@ -109,6 +115,7 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()
 	}
 
 	return w, func() {
+		w.Close() // stop any started background GC goroutine.
 		os.RemoveAll(fn)
 	}
 }
@@ -118,8 +125,8 @@ func TestBadger_SetBitmap(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
 	bitvalue := uint64(0)
 	changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
 	if changed <= 0 {
@@ -140,7 +147,7 @@ func TestBadger_SetBitmap(t *testing.T) {
 	// commited, so should be visible outside the txn
 	//
 
-	tx2 := dbwrap.NewBadgerTx(!writable)
+	tx2 := dbwrap.NewBadgerTx(!writable, index)
 	exists, err = tx2.Contains(index, field, view, shard, bitvalue)
 	panicOn(err)
 	if !exists {
@@ -159,9 +166,9 @@ func TestBadger_OffsetRange(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+
 	bitvalue := uint64(1 << 20)
 	changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
 	if changed <= 0 {
@@ -194,7 +201,7 @@ func TestBadger_OffsetRange(t *testing.T) {
 	start := uint64(0 << 16)
 	endx := bitvalue + 1<<16
 
-	tx2 := dbwrap.NewBadgerTx(!writable)
+	tx2 := dbwrap.NewBadgerTx(!writable, index)
 	rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx)
 	panicOn(err)
 	tx2.Rollback()
@@ -208,7 +215,7 @@ func TestBadger_OffsetRange(t *testing.T) {
 
 	// now offset by 2M
 	offset = uint64(2 << 20)
-	tx3 := dbwrap.NewBadgerTx(!writable)
+	tx3 := dbwrap.NewBadgerTx(!writable, index)
 	rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx)
 	panicOn(err)
 	tx3.Rollback()
@@ -236,7 +243,7 @@ func TestBadger_Count_on_many_containers(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 	}
 
-	tx := dbwrap.NewBadgerTx(writable)
+	tx := dbwrap.NewBadgerTx(writable, index)
 	defer tx.Rollback()
 
 	n, err := tx.Count(index, field, view, shard)
@@ -252,7 +259,7 @@ func TestBadger_Count_dense_containers(t *testing.T) {
 	defer dbwrap.Close()
 	index, field, view, shard := "i", "f", "v", uint64(0)
 
-	tx := dbwrap.NewBadgerTx(writable)
+	tx := dbwrap.NewBadgerTx(writable, index)
 
 	expected := 0
 	// can't do more than about 100k writes per badger txn by default, so
@@ -280,9 +287,9 @@ func TestBadger_ContainerIterator_on_empty(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(!writable)
-	defer tx.Rollback()
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(!writable, index)
+	defer tx.Rollback()
 	bitvalue := uint64(0)
 	citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue)
 	panicOn(err)
@@ -298,9 +305,9 @@ func TestBadger_ContainerIterator_on_one_bit(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-	defer tx.Rollback()
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+	defer tx.Rollback()
 
 	bitvalue := uint64(42)
 
@@ -398,9 +405,9 @@ func TestBadger_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-	defer tx.Rollback()
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+	defer tx.Rollback()
 
 	putme := uint64(1<<16) + 3 // in the key:1 container
 	searchme := putme + 1
@@ -453,9 +460,9 @@ func TestBadger_ContainerIterator_empty_iteration_loop(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_empty_iteration_loop")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-	defer tx.Rollback()
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+	defer tx.Rollback()
 
 	putme := uint64(1<<16) + 3  // in the key:1 container
 	searchme := uint64(1 << 17) // in the next container, key:2
@@ -503,9 +510,9 @@ func TestBadger_ForEach_on_one_bit(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-	defer tx.Rollback()
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+	defer tx.Rollback()
 
 	bitvalue := uint64(42)
 
@@ -563,7 +570,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 
 		// delete, but rollback instead of commit
-		tx := dbwrap.NewBadgerTx(writable)
+		tx := dbwrap.NewBadgerTx(writable, index)
 		hi := highbits(putme)
 		panicOn(tx.RemoveContainer(index, field, view, shard, hi))
 		tx.Rollback()
@@ -572,7 +579,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 
 		// c) within one Tx, after delete it should be gone as viewed within the txn.
-		tx = dbwrap.NewBadgerTx(writable)
+		tx = dbwrap.NewBadgerTx(writable, index)
 		hi = highbits(putme)
 
 		exists, err := tx.Contains(index, field, view, shard, putme)
@@ -624,7 +631,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 
 		// delete, but rollback instead of commit
-		tx := dbwrap.NewBadgerTx(writable)
+		tx := dbwrap.NewBadgerTx(writable, index)
 		hi, lo := highbits(putme), lowbits(putme)
 		_, _ = hi, lo
 		_, err := tx.Remove(index, field, view, shard, hi)
@@ -635,7 +642,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 
 		// c) within one Tx, after delete it should be gone as viewed within the txn.
-		tx = dbwrap.NewBadgerTx(writable)
+		tx = dbwrap.NewBadgerTx(writable, index)
 
 		exists, err := tx.Contains(index, field, view, shard, putme)
 		panicOn(err)
@@ -724,7 +731,7 @@ func TestBadger_Max_on_many_containers(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 	}
 
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, index)
 	defer tx.Rollback()
 
 	max, err := tx.Max(index, field, view, shard)
@@ -742,7 +749,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) {
 	index, field, view, shard := "i", "f", "v", uint64(0)
 
 	// verify no containers flag works
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, index)
 	min, containersExist, err := tx.Min(index, field, view, shard)
 	_ = min
 	panicOn(err)
@@ -759,7 +766,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 	}
 
-	tx = dbwrap.NewBadgerTx(!writable)
+	tx = dbwrap.NewBadgerTx(!writable, index)
 	defer tx.Rollback()
 
 	min, containersExist, err = tx.Min(index, field, view, shard)
@@ -780,7 +787,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) {
 	index, field, view, shard := "i", "f", "v", uint64(0)
 
 	// verify no containers flag works
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, index)
 	n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
 	panicOn(err)
 	if n != 0 {
@@ -796,7 +803,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 	}
 
-	tx = dbwrap.NewBadgerTx(!writable)
+	tx = dbwrap.NewBadgerTx(!writable, index)
 	defer tx.Rollback()
 
 	n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
@@ -824,7 +831,7 @@ func TestBadger_CountRange_middle_container(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 	}
 
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, index)
 	defer tx.Rollback()
 
 	// pick out just the middle container with the 1 bit set on it.
@@ -849,7 +856,7 @@ func TestBadger_CountRange_many_middle_container(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 	}
 
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, index)
 	defer tx.Rollback()
 
 	// get them all
@@ -880,7 +887,7 @@ func TestBadger_UnionInPlace(t *testing.T) {
 		badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
 	}
 
-	tx2 := dbwrap.NewBadgerTx(!writable)
+	tx2 := dbwrap.NewBadgerTx(!writable, index)
 	n, err := tx2.Count(index, field, view, shard)
 	panicOn(err)
 	if n != 2 {
@@ -895,7 +902,7 @@ func TestBadger_UnionInPlace(t *testing.T) {
 	}
 	mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container
 
-	tx := dbwrap.NewBadgerTx(writable)
+	tx := dbwrap.NewBadgerTx(writable, index)
 	defer tx.Rollback()
 	err = tx.UnionInPlace(index, field, view, shard, others, others2, others3)
 	panicOn(err)
@@ -920,7 +927,7 @@ func TestBadger_RoaringBitmap(t *testing.T) {
 	putme := expected
 	badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
 
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, index)
 	defer tx.Rollback()
 
 	rbm, err := tx.RoaringBitmap(index, field, view, shard)
@@ -951,10 +958,8 @@ func TestBadger_reverse_badger_iterator_and_prefix_valid(t *testing.T) {
 		return nil
 	})
 	panicOn(err)
-	//vv("stringifiedBadgerKeys(db) = '%v'", stringifiedBadgerKeys(dbwrap.db))
-	// allkeys:["a:0", "a:1", "a:2", "b:0", "b:1", "b:2", "c:0", "c:1", "c:2", ]'
 
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, "no-index-avail")
 
 	prefix := []byte("b:")
 	it := NewBadgerIterator(tx, prefix)
@@ -1014,10 +1019,8 @@ func TestBadger_just_reverse_badger_iterator_and_prefix_valid(t *testing.T) {
 		return nil
 	})
 	panicOn(err)
-	//vv("stringifiedBadgerKeys(db) = '%v'", stringifiedBadgerKeys(dbwrap.db))
-	// allkeys:["a:0", "a:1", "a:2", "b:0", "b:1", "b:2", "c:0", "c:1", "c:2", ]'
 
-	tx := dbwrap.NewBadgerTx(!writable)
+	tx := dbwrap.NewBadgerTx(!writable, "no-index-avail")
 
 	seekto := []byte("c:")
 	prefix := []byte("b:")
@@ -1046,9 +1049,9 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-	defer tx.Rollback()
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+	defer tx.Rollback()
 
 	//bitvalue := uint64(42)
 
@@ -1062,7 +1065,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
 	clear := false
 	logme := false
 
-	changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
+	changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
 	_ = rowSet
 	if changed != len(bits) {
 		panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
@@ -1079,7 +1082,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
 
 	// now test the union in place with the same set gives no change.
 
-	changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
+	changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
 	_ = rowSet
 	if changed != 0 {
 		panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@@ -1103,7 +1106,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
 		itr, err := roaring.NewRoaringIterator(data)
 		panicOn(err)
 
-		changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
+		changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
 		_ = rowSet
 		if changed != 1 {
 			panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@@ -1128,9 +1131,9 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_set_nonoverlapping_bits")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-	defer tx.Rollback()
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+	defer tx.Rollback()
 
 	// get some roaring bits, get an itr RoaringIterator from them
 	rowSize := uint64(0)
@@ -1148,7 +1151,7 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
 	clear := false
 	logme := false
 
-	changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
+	changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
 	_ = rowSet
 	if changed != len(bits) {
 		panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
@@ -1165,7 +1168,7 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
 
 	// now import the 2nd, overlapping set and set them.
 
-	changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize)
+	changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil)
 	_ = rowSet
 	if changed != 4 {
 		panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@@ -1178,9 +1181,9 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_clear_nonoverlapping_bits")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-	defer tx.Rollback()
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+	defer tx.Rollback()
 
 	// get some roaring bits, get an itr RoaringIterator from them
 	rowSize := uint64(0)
@@ -1198,7 +1201,7 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
 	clear := false
 	logme := false
 
-	changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
+	changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
 	_ = rowSet
 	if changed != len(bits) {
 		panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
@@ -1216,7 +1219,7 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
 	// now import the 2nd overlapping set and clear them.
 	clear = true
 
-	changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize)
+	changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil)
 	_ = rowSet
 	if changed != 2 {
 		panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@@ -1252,9 +1255,9 @@ func TestBadger_DeleteIndex(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-	bitvalue := uint64(777)
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+	bitvalue := uint64(777)
 	bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
 	for _, v := range bits {
 		changed, err := tx.Add(index, field, view, shard, doBatched, v)
@@ -1290,7 +1293,7 @@ func TestBadger_DeleteIndex(t *testing.T) {
 	err = dbwrap.DeleteIndex(index)
 	panicOn(err)
 
-	tx = dbwrap.NewBadgerTx(!writable)
+	tx = dbwrap.NewBadgerTx(!writable, index2)
 	defer tx.Rollback()
 	exists, err = tx.Contains(index2, field, view, shard, bitvalue)
 	panicOn(err)
@@ -1314,9 +1317,9 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
 	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex_over100k")
 	defer clean()
 	defer dbwrap.Close()
-	tx := dbwrap.NewBadgerTx(writable)
-	bitvalue := uint64(777)
 	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+	bitvalue := uint64(777)
 	limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction.
 	//limit := uint64(101)
 	for v := uint64(1); v < limit; v++ {
@@ -1328,7 +1331,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
 		panicOn(err)
 		if v%100000 == 0 {
 			panicOn(tx.Commit())
-			tx = dbwrap.NewBadgerTx(writable)
+			tx = dbwrap.NewBadgerTx(writable, index)
 		}
 	}
 
@@ -1345,7 +1348,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
 	err = dbwrap.DeleteIndex(index)
 	panicOn(err)
 
-	tx = dbwrap.NewBadgerTx(!writable)
+	tx = dbwrap.NewBadgerTx(!writable, index2)
 	defer tx.Rollback()
 	exists, err := tx.Contains(index2, field, view, shard, bitvalue)
 	panicOn(err)
@@ -1445,3 +1448,135 @@ func mustAddR(changed bool, err error) {
 func mustRemove(changeCount int, err error) {
 	panicOn(err)
 }
+
+func TestBadger_DeleteFragment(t *testing.T) {
+
+	// setup
+	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteFragment")
+	defer clean()
+	defer dbwrap.Close()
+	index, field, view, shard0 := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index)
+
+	shard1 := uint64(1)
+
+	bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
+	shards := []uint64{shard0, shard1}
+	for _, s := range shards {
+		for _, v := range bits {
+			changed, err := tx.Add(index, field, view, s, doBatched, v)
+			if changed <= 0 {
+				panic("should have changed")
+			}
+			panicOn(err)
+		}
+	}
+
+	for _, s := range shards {
+		for _, v := range bits {
+			exists, err := tx.Contains(index, field, view, s, v)
+			panicOn(err)
+			if !exists {
+				panic("ARG bitvalue was NOT SET!!!")
+			}
+		}
+	}
+	err := tx.Commit()
+	panicOn(err)
+	//vv("Dump: %v", dbwrap.StringifiedBadgerKeys(nil))
+
+	// end of setup
+
+	survivor := shard0
+	victim := shard1
+	err = dbwrap.DeleteFragment(index, field, view, victim, nil)
+	panicOn(err)
+
+	tx = dbwrap.NewBadgerTx(!writable, index)
+	defer tx.Rollback()
+
+	for _, s := range shards {
+		for _, v := range bits {
+			exists, err := tx.Contains(index, field, view, s, v)
+			panicOn(err)
+			if s == survivor {
+				if !exists {
+					panic(fmt.Sprintf("ARG survivor died : bit %v", v))
+				}
+			} else if s == victim { // victim, should have been deleted
+				if exists {
+					panic(fmt.Sprintf("ARG victim lived : bit %v", v))
+				}
+			}
+		}
+	}
+}
+
+func TestBadger_shardFromBadgerKey(t *testing.T) {
+	if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615")) != 1 {
+		panic("problem")
+	}
+	if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615")) != 0 {
+		panic("problem")
+	}
+	if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@18446744073709551615")) != 18446744073709551615 {
+		panic("problem")
+	}
+
+	func() {
+		defer func() {
+			r := recover()
+			if r == nil {
+				panic("should have panic-ed")
+			}
+		}()
+		// called for the panic of a short ckey, only 19 bytes instead of 20
+		shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@1844674407370955161"))
+	}()
+
+}
+
+func TestBadger_SliceOfShards(t *testing.T) {
+
+	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SliceOfShards")
+	defer clean()
+	defer dbwrap.Close()
+	index, field, view := "i", "f", "v"
+	shards := []uint64{0, 1, 2, 3, 1000001, 2000001}
+	putme := uint64(179)
+	for _, shard := range shards {
+		badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
+	}
+	tx := dbwrap.NewBadgerTx(!writable, index)
+	defer tx.Rollback()
+
+	slc, err := tx.SliceOfShards(index, field, view, "")
+	panicOn(err)
+	for i := range shards {
+		if shards[i] != slc[i] {
+			panic(fmt.Sprintf("expected at i=%v that slc[i]=%v = shards[i]=%v", i, slc[i], shards[i]))
+		}
+	}
+}
+
+func reportTestBadgersNeedingClose() {
+	globalBadgerReg.mu.Lock()
+	defer globalBadgerReg.mu.Unlock()
+	n := len(globalBadgerReg.mp)
+	if n > 0 {
+		AlwaysPrintf("*** these badgers are still open (n=%v):", n)
+		i := 0
+		for w := range globalBadgerReg.mp {
+			AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack)
+			i++
+		}
+	}
+}
+
+var _ = reportTestBadgersNeedingClose // happy linter
+
+func TestMain(m *testing.M) {
+	ret := m.Run()
+	//reportTestBadgersNeedingClose()
+	os.Exit(ret)
+}
diff --git a/bluegreentx.go b/bluegreentx.go
index dd58c69fd..3df87577a 100644
--- a/bluegreentx.go
+++ b/bluegreentx.go
@@ -16,6 +16,9 @@ package pilosa
 
 import (
 	"fmt"
+	"io"
+	"reflect"
+	"sort"
 
 	"github.com/pilosa/pilosa/v2/roaring"
 )
@@ -27,6 +30,8 @@ type blueGreenTx struct {
 	b Tx // b's output is returned
 
 	idx *Index
+
+	checker blueGreenChecker
 }
 
 func newBlueGreenTx(a, b Tx, idx *Index) *blueGreenTx {
@@ -37,6 +42,10 @@ var _ = newBlueGreenTx // keep linter happy
 
 var _ Tx = (*blueGreenTx)(nil)
 
+func (c *blueGreenTx) Type() string {
+	return c.a.Type() + "_" + c.b.Type()
+}
+
 func (c *blueGreenTx) Readonly() bool {
 	a := c.a.Readonly()
 	b := c.b.Readonly()
@@ -47,6 +56,7 @@ func (c *blueGreenTx) Readonly() bool {
 }
 
 func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
+	c.checker.see(index, field, view, shard)
 	return c.b.NewTxIterator(index, field, view, shard)
 }
 
@@ -55,11 +65,71 @@ func (c *blueGreenTx) Pointer() string {
 }
 
 func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
+	c.checker.see(index, field, view, shard)
 	c.a.IncrementOpN(index, field, view, shard, changedN)
 	c.b.IncrementOpN(index, field, view, shard, changedN)
 }
 
+func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
+	here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard)
+	aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0)
+	bIter, bFound, bErr := c.b.ContainerIterator(index, field, view, shard, 0)
+
+	if aFound != bFound {
+		panic(fmt.Sprintf("compareTxState[%v]: A ContainerIterator had aFound=%v, but B had bFound=%v; at '%v'", here, aFound, bFound, stack()))
+	}
+
+	if aErr == nil {
+		defer aIter.Close()
+	}
+	if bErr == nil {
+		defer bIter.Close()
+	}
+	if aErr != nil || bErr != nil {
+		if aErr != nil && bErr != nil {
+			panic(fmt.Sprintf("compareTxState[%v]: A reported err '%v'; B reported err '%v' at %v", here, aErr, bErr, stack()))
+		}
+		if aErr != nil {
+			panic(fmt.Sprintf("compareTxState[%v]: A reported err %v at %v; but B did not", here, aErr, stack()))
+		}
+		if bErr != nil {
+			panic(fmt.Sprintf("compareTxState[%v]: B reported err %v at %v; but A did not", here, bErr, stack()))
+		}
+	}
+	for aIter.Next() {
+		aKey, aValue := aIter.Value()
+		if !bIter.Next() {
+			panic(fmt.Sprintf("compareTxState[%v]: A found key %v, B didn't, at %v", here, aKey, stack()))
+		}
+		bKey, bValue := bIter.Value()
+		if bKey != aKey {
+			panic(fmt.Sprintf("compareTxState[%v]: A found key %v, B found %v, at %v", here, aKey, bKey, stack()))
+		}
+		if err := aValue.BitwiseCompare(bValue); err != nil {
+			panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v at %v", here, aKey, err, stack()))
+		}
+	}
+	// end checking everything in A, but does B have more?
+	if bIter.Next() {
+		bKey, _ := bIter.Value()
+		panic(fmt.Sprintf("compareTxState[%v]: B found key %v, A didn't, at %v", here, bKey, stack()))
+	}
+}
+
+func (c *blueGreenTx) checkDatabase() {
+	for index, fields := range c.checker.seen() {
+		for field, views := range fields {
+			for view, shards := range views {
+				for shard := range shards {
+					c.compareTxState(index, field, view, shard)
+				}
+			}
+		}
+	}
+}
+
 func (c *blueGreenTx) Rollback() {
+	c.checkDatabase()
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
@@ -71,6 +141,7 @@ func (c *blueGreenTx) Rollback() {
 }
 
 func (c *blueGreenTx) Commit() error {
+	c.checkDatabase()
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
@@ -86,6 +157,7 @@ func (c *blueGreenTx) Commit() error {
 }
 
 func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
@@ -100,6 +172,7 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r
 }
 
 func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
@@ -116,6 +189,7 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin
 }
 
 func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
@@ -126,17 +200,11 @@ func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key
 	errB := c.b.PutContainer(index, field, view, shard, key, rc)
 	compareErrors(errA, errB)
 
-	/*  draft idea of how to check the full databases afterwards:
-	hashA := c.a.RootHashString()
-	hashB := c.b.RootHashString()
-	if hashA != hashB {
-		panic(fmt.Sprintf("hashA = '%v' but hashB = '%v'", hashA, hashB))
-	}
-	*/
 	return errB
 }
 
-func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
+func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
@@ -147,49 +215,37 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
 	// remember where the iterator started, so we can replay it a second time.
 	rit2 := rit.Clone()
 
-	changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
+	changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
 
-	changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize)
+	changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data)
 
-	if changedA != changedB {
-		panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
-	}
-	if len(rowSetA) != len(rowSetB) {
-		panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
-	}
-	for k, va := range rowSetA {
-		vb, ok := rowSetB[k]
-		if !ok {
-			panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
+	if len(data) == 0 {
+		// okay to check! otherwise we are in the fragment.fillFragmentFromArchive
+		// case where we know that RoaringTx.ImportRoaringBits changed and rowSet will
+		// be inaccurate.
+		if changedA != changedB {
+			panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
 		}
-		if va != vb {
-			panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
+		if len(rowSetA) != len(rowSetB) {
+			panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
+		}
+		for k, va := range rowSetA {
+			vb, ok := rowSetB[k]
+			if !ok {
+				panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
+			}
+			if va != vb {
+				panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
+			}
 		}
 	}
-
 	compareErrors(errA, errB)
 
-	//compareDatabases(c.a, c.b)
 	return changedB, rowSetB, errB
 }
 
-/* // TODO: get a database-wide checksum working
-func compareDatabases(a, b Tx) {
-
-	index, field, view, shard := "i", "f", "v", uint64(0)
-
-	ha, errA := a.WholeDatabaseBlake3Hash(index, field, view, shard)
-	panicOn(errA)
-	hb, errB := b.WholeDatabaseBlake3Hash(index, field, view, shard)
-	panicOn(errB)
-
-	if ha != hb {
-		panic(fmt.Sprintf("a.WholeDatabaseBlake3Hash(%T) = '%v' but b.WholeDatabaseBlake3Hash(%T) = '%v'", a, ha, b, hb))
-	}
-}
-*/
-
 func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
@@ -207,6 +263,7 @@ func (c *blueGreenTx) UseRowCache() bool {
 }
 
 func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see Add() panic '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, stack())
@@ -249,6 +306,7 @@ func compareErrors(errA, errB error) {
 }
 
 func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
@@ -263,6 +321,7 @@ func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint6
 }
 
 func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
@@ -278,6 +337,7 @@ func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint
 }
 
 func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
@@ -290,10 +350,14 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64,
 	bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
 
 	compareErrors(errA, errB)
+	if errA != nil {
+		ait.Close() // don't leak it.
+	}
 	return bit, bfound, errB
 }
 
 func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
@@ -310,6 +374,7 @@ func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i
 }
 
 func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
@@ -326,6 +391,7 @@ func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, star
 }
 
 func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
@@ -342,6 +408,7 @@ func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, er
 }
 
 func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
@@ -358,6 +425,7 @@ func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, erro
 }
 
 func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
@@ -374,6 +442,7 @@ func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool
 }
 
 func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
@@ -387,6 +456,7 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe
 }
 
 func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
@@ -405,6 +475,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
 }
 
 func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
+	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
@@ -419,3 +490,136 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star
 	compareErrors(errA, errB)
 	return b, errB
 }
+
+func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+	c.checker.see(index, field, view, shard)
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+
+	rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
+	rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
+	if szA != szB {
+		panic(fmt.Sprintf("szA = %v, but szB = %v", szA, szB))
+	}
+	compareErrors(errA, errB)
+	return &MultiReaderB{a: rcA, b: rcB}, szB, errB
+}
+
+func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
+	// doesn't change state, so we don't really need see() call here. And we don't have a single shard for it.
+	//c.checker.see(index, field, view, shard) // don't have shard.
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	slcA, errA := c.a.SliceOfShards(index, field, view, optionalViewPath)
+	slcB, errB := c.b.SliceOfShards(index, field, view, optionalViewPath)
+	compareErrors(errA, errB)
+
+	// sort order may be different, and that's ok.
+	cpa := append([]uint64{}, slcA...)
+	cpb := append([]uint64{}, slcB...)
+	sort.Slice(cpa, func(i, j int) bool { return cpa[i] < cpa[j] })
+	sort.Slice(cpb, func(i, j int) bool { return cpb[i] < cpb[j] })
+
+	if !reflect.DeepEqual(cpa, cpb) {
+		// report the first difference
+		ma := make(map[uint64]bool)
+		for _, ka := range slcA {
+			ma[ka] = true
+		}
+		for _, kb := range slcB {
+			if !ma[kb] {
+				panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B had %v, but A did not; in the SliceOfShards returned slice.", kb))
+			}
+			delete(ma, kb)
+		}
+		if len(ma) != 0 {
+			for _, firstDifference := range ma {
+				panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A had %v, but B did not; in the SliceOfShards returned slice.", firstDifference))
+			}
+		}
+		panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA='%#v';\n slcB='%#v';\n", cpa, cpb))
+	}
+	return slcB, errB
+}
+
+type MultiReaderB struct {
+	a io.ReadCloser
+	b io.ReadCloser
+}
+
+// TODO(jea): test this for accuracy/correctness.
+func (m *MultiReaderB) Read(p []byte) (nB int, errB error) {
+	nB, errB = m.b.Read(p)
+	p2 := make([]byte, nB)
+	// discard the exact same amount from A
+	// ReadAtLeast reads from r into buf until it has read at least
+	// min bytes. It returns the number of bytes copied and an error
+	// if fewer bytes were read. The error is EOF only if no bytes
+	// were read. If an EOF happens after reading fewer than min bytes,
+	// ReadAtLeast returns ErrUnexpectedEOF. If min is greater than
+	// the length of buf, ReadAtLeast returns ErrShortBuffer. On
+	// return, n >= min if and only if err == nil. If r returns
+	// an error having read at least min bytes, the error is dropped.
+	nA, errA := io.ReadAtLeast(m.a, p2, nB)
+	if errA == io.ErrUnexpectedEOF {
+		panic(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA))
+	}
+	if nA != nB {
+		panic(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA))
+	}
+	return
+}
+
+func (m *MultiReaderB) Close() error {
+	m.a.Close()
+	return m.b.Close()
+}
+
+// blueGreenChecker is used
+type blueGreenChecker struct {
+	visited map[string]map[string]map[string]map[uint64]struct{}
+	done    bool
+}
+
+// see would mark a thing as seen.
+func (b *blueGreenChecker) see(index, field, view string, shard uint64) {
+	if b.visited == nil {
+		b.visited = make(map[string]map[string]map[string]map[uint64]struct{})
+	}
+	var visitedIdx map[string]map[string]map[uint64]struct{}
+	var visitedField map[string]map[uint64]struct{}
+	var visitedView map[uint64]struct{}
+
+	if visitedIdx = b.visited[index]; visitedIdx == nil {
+		visitedIdx = make(map[string]map[string]map[uint64]struct{})
+		b.visited[index] = visitedIdx
+	}
+	if visitedField = visitedIdx[field]; visitedField == nil {
+		visitedField = make(map[string]map[uint64]struct{})
+		visitedIdx[field] = visitedField
+	}
+	if visitedView = visitedField[view]; visitedView == nil {
+		visitedView = make(map[uint64]struct{})
+		visitedField[view] = visitedView
+	}
+	visitedView[shard] = struct{}{}
+}
+
+// seen reports the things it has seen, exactly once so
+// that Rollback can be called after Commit without repeating
+// the check.
+func (b *blueGreenChecker) seen() map[string]map[string]map[string]map[uint64]struct{} {
+	if b.done {
+		return nil
+	}
+	b.done = true
+	return b.visited
+}
diff --git a/catcher.go b/catcher.go
index a4d832c20..5623733c2 100644
--- a/catcher.go
+++ b/catcher.go
@@ -16,6 +16,7 @@ package pilosa
 
 import (
 	"fmt"
+	"io"
 
 	"github.com/pilosa/pilosa/v2/roaring"
 )
@@ -51,14 +52,14 @@ func (c *catcherTx) WholeDatabaseBlake3Hash(index, field, view string, shard uin
 	return c.b.WholeDatabaseBlake3Hash(index, field, view, shard)
 }
 
-func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
+func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
 			panic(r)
 		}
 	}()
-	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
+	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
 }
 
 func (c *catcherTx) Readonly() bool {
@@ -275,3 +276,26 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start,
 	}()
 	return c.b.OffsetRange(index, field, view, shard, offset, start, end)
 }
+
+func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
+}
+
+func (c *catcherTx) Type() string {
+	return c.b.Type()
+}
+func (c *catcherTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
+			panic(r)
+		}
+	}()
+	return c.b.SliceOfShards(index, field, view, optionalViewPath)
+}
diff --git a/cluster.go b/cluster.go
index ed1d92eb9..eb0cba364 100644
--- a/cluster.go
+++ b/cluster.go
@@ -525,7 +525,7 @@ func (c *cluster) unprotectedSetState(state string) {
 		cleaner.Cluster = c
 		cleaner.Closing = c.closing
 
-		// Clean holder.
+		// Clean holder. This is where the shard gets removed after resize.
 		if err := cleaner.CleanHolder(); err != nil {
 			c.logger.Printf("holder clean error: err=%s", err)
 		}
diff --git a/cluster_internal_test.go b/cluster_internal_test.go
index c31b9f294..1c344da06 100644
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@@ -96,7 +96,9 @@ func newIndexWithTempPath(name string) *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(NewHolder(DefaultPartitionN), path, name)
+	h := NewHolder(DefaultPartitionN)
+	h.Path = path
+	index, err := h.CreateIndex(name, IndexOptions{})
 	if err != nil {
 		panic(err)
 	}
@@ -160,7 +162,7 @@ func TestFragSources(t *testing.T) {
 	defer idx.Close()
 
 	// Obtain transaction.
-	tx := &RoaringTx{Index: idx}
+	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx})
 	defer tx.Rollback()
 
 	field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
@@ -809,12 +811,12 @@ func TestCluster_ResizeStates(t *testing.T) {
 		if idx0 == nil {
 			t.Fatal(`idx0 was nil, could not retrieve Index("i")`)
 		}
-		//idx0.Dump("node0")
 
 		// addNode needs to block until the resize process has completed.
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
+
 		node1 := tc.Clusters[1]
 
 		// Ensure that nodes come up in state NORMAL.
@@ -846,8 +848,6 @@ func TestCluster_ResizeStates(t *testing.T) {
 		if idx1 == nil {
 			t.Fatal(`idx1 was nil, could not retrieve Index("i")`)
 		}
-		//idx0.Dump("after rebalance, node0")
-		//idx1.Dump("after rebalance, node1")
 
 		// Ensure checksums are the same.
 		if chksum, err := node1Fragment.Checksum(); err != nil {
@@ -872,6 +872,7 @@ func TestAE(t *testing.T) {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leaking a goroutine.
 		select {
 		case <-ch:
 			return
@@ -883,11 +884,13 @@ func TestAE(t *testing.T) {
 	t.Run("AbortBlocksInitialized", func(t *testing.T) {
 		c := newCluster()
 		c.initializeAntiEntropy()
+
 		ch := make(chan struct{})
 		go func() {
 			c.abortAntiEntropy()
 			close(ch)
 		}()
+		defer c.abortAntiEntropyQ() // avoid leak of goroutine.
 		select {
 		case <-ch:
 			t.Fatalf("aborting anti entropy on an initialized cluster didn't block")
diff --git a/executor.go b/executor.go
index 62b98b5be..6a072372a 100644
--- a/executor.go
+++ b/executor.go
@@ -19,12 +19,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
+	"math/bits"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/molecula/ext"
 	"github.com/pilosa/pilosa/v2/pql"
 	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/roaring"
@@ -64,12 +64,6 @@ type executor struct {
 	workersWG      sync.WaitGroup
 	workerPoolSize int
 	work           chan job
-	// global registry to check for name clashes
-	additionalOps map[string]*ext.BitmapOp
-	// typed registries we can use in lookups
-	additionalBitmapOps map[string]ext.BitmapOpBitmap
-	additionalCountOps  map[string]ext.BitmapOpUnaryCount
-	additionalFieldOps  map[string]ext.BitmapOpBSIBitmap
 }
 
 // executorOption is a functional option type for pilosa.Executor
@@ -125,38 +119,6 @@ func (e *executor) Close() error {
 	return nil
 }
 
-func (e *executor) registerOps(ops []ext.BitmapOp) error {
-	if e.additionalOps == nil {
-		e.additionalOps = make(map[string]*ext.BitmapOp)
-		e.additionalBitmapOps = make(map[string]ext.BitmapOpBitmap)
-		e.additionalCountOps = make(map[string]ext.BitmapOpUnaryCount)
-		e.additionalFieldOps = make(map[string]ext.BitmapOpBSIBitmap)
-	}
-	for i, op := range ops {
-		name := op.Name
-		if _, exists := e.additionalOps[name]; exists {
-			return fmt.Errorf("op name '%s' already defined", name)
-		}
-		e.additionalOps[name] = &ops[i]
-		typ := ops[i].Func.BitmapOpType()
-		switch {
-		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
-			e.additionalCountOps[name] = ops[i].Func.(ext.BitmapOpUnaryCount)
-		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
-			e.additionalBitmapOps[name] = ops[i].Func.(ext.BitmapOpBitmap)
-		case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
-			if fn, ok := ops[i].Func.(ext.BitmapOpBSIBitmapPrecall); ok {
-				e.additionalFieldOps[name] = ext.BitmapOpBSIBitmap(fn)
-			} else {
-				e.additionalFieldOps[name] = ops[i].Func.(ext.BitmapOpBSIBitmap)
-			}
-		default:
-			return fmt.Errorf("unsupported types for '%s': input type %d, output type %d", name, typ.Input, typ.Output)
-		}
-	}
-	return nil
-}
-
 // Execute executes a PQL query.
 func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute")
@@ -228,7 +190,6 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
 	} else if err := validateQueryContext(ctx); err != nil {
 		return resp, err
 	}
-
 	resp.Results = results
 
 	// Fill column attributes if requested.
@@ -369,7 +330,6 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr
 // handlePreCalls traverses the call tree looking for calls that need
 // precomputed values. Right now, that's just Distinct.
 func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
-
 	if c.Name == "Precomputed" {
 		idx := c.Args["valueidx"].(int64)
 		if idx >= 0 && idx < int64(len(opt.EmbeddedData)) {
@@ -404,18 +364,17 @@ func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *p
 	// like Distinct, where you can't predict output shard for a result
 	// from the shard being queried.
 	if newIndex != "" && newIndex != index {
-		if err := e.handlePreCallChildren(ctx, tx, index, c, shards, opt); err != nil {
-			return err
-		}
-
 		c.Type = pql.PrecallGlobal
 		index = newIndex
 		// we need to recompute shards, then
 		shards = nil
 	}
+	if err := e.handlePreCallChildren(ctx, tx, index, c, shards, opt); err != nil {
+		return err
+	}
+	// child calls already handled, no precall for this, so we're done
 	if c.Type == pql.PrecallNone {
-		// otherwise, handle the children
-		return e.handlePreCallChildren(ctx, tx, index, c, shards, opt)
+		return nil
 	}
 	// We don't try to handle sub-calls from here. I'm not 100%
 	// sure that's right, but I think the fact that they're happening
@@ -548,6 +507,7 @@ func (e *executor) execute(ctx context.Context, tx Tx, index string, q *pql.Quer
 		if err != nil {
 			return nil, err
 		}
+
 		results = append(results, v)
 		// Some Calls can have significant data associated with them
 		// that gets generated during processing, such as Precomputed
@@ -708,15 +668,6 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql.
 		return nil, err
 	}
 
-	// Special handling for mutation and top-n calls.
-	if op, ok := e.additionalCountOps[c.Name]; ok {
-		statFn()
-		return e.executeGenericCount(ctx, tx, index, c, op, shards, opt)
-	}
-	if op, ok := e.additionalFieldOps[c.Name]; ok {
-		statFn()
-		return e.executeGenericField(ctx, tx, index, c, op, shards, opt)
-	}
 	switch c.Name {
 	case "Sum":
 		statFn()
@@ -739,6 +690,9 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql.
 	case "ClearRow":
 		statFn()
 		return e.executeClearRow(ctx, tx, index, c, shards, opt)
+	case "Distinct":
+		statFn()
+		return e.executeDistinct(ctx, tx, index, c, shards, opt)
 	case "Store":
 		statFn()
 		return e.executeSetRow(ctx, tx, index, c, shards, opt)
@@ -1151,11 +1105,9 @@ func (e *executor) executeSum(ctx context.Context, tx Tx, index string, c *pql.C
 	return other, nil
 }
 
-// executeGenericField executes a generic call on a field. Note that in this
-// implementation, the operation is always a BSI op.
-func (e *executor) executeGenericField(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) {
-	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericField")
-	span.LogKV("name", c.Name)
+// executeDistinct executes a Distinct call on a field.
+func (e *executor) executeDistinct(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (SignedRow, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct")
 	defer span.Finish()
 
 	field := c.Args["field"]
@@ -1165,7 +1117,7 @@ func (e *executor) executeGenericField(ctx context.Context, tx Tx, index string,
 
 	// Execute calls in bulk on each remote node and merge.
 	mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
-		return e.executeGenericFieldShard(ctx, tx, index, c, op, shard)
+		return e.executeDistinctShard(ctx, tx, index, c, shard)
 	}
 
 	// Merge returned results at coordinating node.
@@ -1440,13 +1392,6 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard")
 	defer span.Finish()
 
-	if _, ok := e.additionalCountOps[c.Name]; ok {
-		return nil, fmt.Errorf("count op %s used as bitmap call", c.Name)
-	}
-	if op, ok := e.additionalBitmapOps[c.Name]; ok {
-		return e.executeGenericBitmapShard(ctx, tx, index, c, op, shard)
-	}
-
 	switch c.Name {
 	case "Row", "Range":
 		return e.executeRowShard(ctx, tx, index, c, shard)
@@ -1464,6 +1409,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri
 		return e.executeShiftShard(ctx, tx, index, c, shard)
 	case "All": // Allow a shard computation to use All() (note, limit/offset not applied)
 		return e.executeAllCallShard(ctx, tx, index, c, shard)
+	case "Distinct":
+		return nil, errors.New("Distinct shouldn't be hit as a bitmap call")
 	case "Precomputed":
 		return e.executePrecomputedCallShard(ctx, tx, index, c, shard)
 	default:
@@ -1471,11 +1418,10 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri
 	}
 }
 
-// executeGenericFieldShard executes a generic/extension command on a
-// single shard. Note that in this implementation, the op is always
-// a BSI op.
-func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) {
-	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericShard")
+// executeDistinctShard executes a Distinct call on a single shard, yielding
+// a SignedRow of the values found.
+func (e *executor) executeDistinctShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (result SignedRow, err error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinctShard")
 	defer span.Finish()
 
 	var filter *Row
@@ -1483,7 +1429,7 @@ func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index st
 	if len(c.Children) == 1 {
 		row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard)
 		if err != nil {
-			return SignedRow{}, errors.Wrap(err, "executing bitmap call")
+			return result, errors.Wrap(err, "executing bitmap call")
 		}
 		filter = row
 		if filter != nil && len(filter.segments) > 0 {
@@ -1497,29 +1443,115 @@ func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index st
 
 	field := e.Holder.Field(index, fieldName)
 	if field == nil {
-		return SignedRow{}, nil
+		return result, nil
 	}
 
 	bsig := field.bsiGroup(fieldName)
 	if bsig == nil {
-		return SignedRow{}, nil
+		return result, nil
+	}
+	view := viewBSIGroupPrefix + fieldName
+
+	depth := uint64(bsig.BitDepth)
+	offset := bsig.Base
+
+	existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*0, ShardWidth*1)
+	if err != nil {
+		return result, err
+	}
+	if filter != nil {
+		existsBitmap = existsBitmap.Intersect(filterBitmap)
+	}
+	if !existsBitmap.Any() {
+		return result, nil
 	}
 
-	fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
-	if fragment == nil {
-		return SignedRow{}, nil
+	signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*1, ShardWidth*2)
+	if err != nil {
+		return result, nil
 	}
 
-	var out ext.SignedBitmap
-	if filterBitmap != nil {
-		out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{WrapBitmap(filterBitmap)}, c.Args)
-	} else {
-		out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{}, c.Args)
+	dataBitmaps := make([]*roaring.Bitmap, depth)
+
+	for i := uint64(0); i < depth; i++ {
+		dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*(i+2), ShardWidth*(i+3))
+		if err != nil {
+			return result, err
+		}
 	}
 
+	// we need spaces for sign bit, existence/filter bit, and data
+	// row bits, which we'll be grabbing 65k bits at a time
+	stashWords := make([]uint64, 1024*(depth+2))
+	bitStashes := make([][]uint64, depth)
+	for i := uint64(0); i < depth; i++ {
+		start := i * 1024
+		last := start + 1024
+		bitStashes[i] = stashWords[start:last]
+		i++
+	}
+	stashOffset := depth * 1024
+	existStash := stashWords[stashOffset : stashOffset+1024]
+	signStash := stashWords[stashOffset+1024 : stashOffset+2048]
+	dataBits := make([][]uint64, depth)
+
+	posValues := make([]uint64, 0, 64)
+	negValues := make([]uint64, 0, 64)
+
+	posBitmap := roaring.NewFileBitmap()
+	negBitmap := roaring.NewFileBitmap()
+
+	existIterator, _ := existsBitmap.Containers.Iterator(0)
+	for existIterator.Next() {
+		key, value := existIterator.Value()
+		if value.N() == 0 {
+			continue
+		}
+		exists := value.AsBitmap(existStash)
+		sign := signBitmap.Containers.Get(key).AsBitmap(signStash)
+		for i := uint64(0); i < depth; i++ {
+			dataBits[i] = dataBitmaps[i].Containers.Get(key).AsBitmap(bitStashes[i])
+		}
+		for idx, word := range exists {
+			// mask holds a mask we can test the other words against.
+			mask := uint64(1)
+			for word != 0 {
+				shift := uint(bits.TrailingZeros64(word))
+				// we shift one *more* than that, to move the
+				// actual one bit off.
+				word >>= shift + 1
+				mask <<= shift
+				value := int64(0)
+				for b := uint64(0); b < depth; b++ {
+					if dataBits[b][idx]&mask != 0 {
+						value += (1 << b)
+					}
+				}
+				if sign[idx]&mask != 0 {
+					value *= -1
+				}
+				value += int64(offset)
+				if value < 0 {
+					negValues = append(negValues, uint64(-value))
+				} else {
+					posValues = append(posValues, uint64(value))
+				}
+				// and now we processed that bit, so we move the mask over one.
+				mask <<= 1
+			}
+			if len(negValues) > 0 {
+				_, _ = negBitmap.AddN(negValues...)
+				negValues = negValues[:0]
+			}
+			if len(posValues) > 0 {
+				_, _ = posBitmap.AddN(posValues...)
+				posValues = posValues[:0]
+			}
+		}
+	}
 	return SignedRow{
-		Neg: NewRowFromBitmap(UnwrapBitmap(out.Neg)),
-		Pos: NewRowFromBitmap(UnwrapBitmap(out.Pos)),
+		Neg: NewRowFromBitmap(negBitmap),
+		Pos: NewRowFromBitmap(posBitmap),
 	}, nil
 }
 
@@ -2681,11 +2713,13 @@ func (e *executor) executeRowsShard(ctx context.Context, tx Tx, index string, fi
 }
 
 func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) {
+
 	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowShard")
 	defer span.Finish()
 
 	// Handle bsiGroup ranges differently.
 	if c.HasConditionArg() {
+		// looks the same on badger/roaring. we think.
 		return e.executeRowBSIGroupShard(ctx, tx, index, c, shard)
 	}
 
@@ -2795,6 +2829,7 @@ func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c *
 
 // executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard.
 func (e *executor) executeRowBSIGroupShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (_ *Row, err error) {
+
 	span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowBSIGroupShard")
 	defer span.Finish()
 
@@ -2975,44 +3010,6 @@ func (e *executor) executeIntersectShard(ctx context.Context, tx Tx, index strin
 	return other, nil
 }
 
-// executeGenericBitmapShard executes a generic bitmap call for a local shard.
-func (e *executor) executeGenericBitmapShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) {
-	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericBitmapShard")
-	defer span.Finish()
-
-	if op.BitmapOpArity() == ext.OpArityUnary {
-		if len(c.Children) != 1 {
-			return nil, fmt.Errorf("%s needs exactly one row parameter", c.Name)
-		}
-		row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard)
-		if err != nil {
-			return nil, err
-		}
-		return row.GenericUnaryOp(op.BitmapOpFunc(), c.Args), nil
-	}
-
-	var err error
-	rows := make([]*Row, len(c.Children))
-	for i, input := range c.Children {
-		rows[i], err = e.executeBitmapCallShard(ctx, tx, index, input, shard)
-		if err != nil {
-			return nil, err
-		}
-	}
-	var other *Row
-	switch op.BitmapOpArity() {
-	case ext.OpArityBinary:
-		other = rows[0]
-		for _, row := range rows[1:] {
-			other = other.GenericBinaryOp(op.BitmapOpFunc(), row, c.Args)
-		}
-	case ext.OpArityNary:
-		other = rows[0].GenericNaryOp(op.BitmapOpFunc(), rows[1:], c.Args)
-	}
-	other.invalidateCount()
-	return other, nil
-}
-
 // executeUnionShard executes a union() call for a local shard.
 func (e *executor) executeUnionShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
@@ -3164,41 +3161,6 @@ func (e *executor) executeShiftShard(ctx context.Context, tx Tx, index string, c
 	return row.Shift(n)
 }
 
-// executeGeneric executes a provided count-like call.
-func (e *executor) executeGenericCount(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) {
-	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericCount")
-	defer span.Finish()
-
-	if len(c.Children) == 0 {
-		return 0, fmt.Errorf("%s() requires an input bitmap", c.Name)
-	} else if len(c.Children) > 1 {
-		return 0, fmt.Errorf("%s() only accepts a single bitmap input", c.Name)
-	}
-
-	// Execute calls in bulk on each remote node and merge.
-	mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
-		row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard)
-		if err != nil {
-			return 0, err
-		}
-		return row.GenericCount(op, c.Args), nil
-	}
-
-	// Merge returned results at coordinating node.
-	reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
-		other, _ := prev.(uint64)
-		return other + v.(uint64)
-	}
-
-	result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
-	if err != nil {
-		return 0, err
-	}
-	n, _ := result.(uint64)
-
-	return n, nil
-}
-
 // executeCount executes a count() call.
 func (e *executor) executeCount(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount")
diff --git a/executor_test.go b/executor_test.go
index 9a78b96c0..73f1d44ce 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -3413,6 +3413,7 @@ func TestExecutor_Execute_Existence(t *testing.T) {
 		defer c.Close()
 		hldr := test.Holder{Holder: c[0].Server.Holder()}
 		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
+
 		_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatal(err)
@@ -3438,6 +3439,7 @@ func TestExecutor_Execute_Existence(t *testing.T) {
 		} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) {
 			t.Fatalf("unexpected columns after Not: %+v", bits)
 		}
+
 		// Reopen cluster to ensure existence field is reloaded.
 		if err := c[0].Reopen(); err != nil {
 			t.Fatal(err)
@@ -4174,25 +4176,6 @@ func TestExecutor_Execute_SetRow(t *testing.T) {
 			t.Fatalf("unexpected columns: %+v", bits)
 		}
 	})
-	t.Run("Err_Store(Distinct)", func(t *testing.T) {
-		c := test.MustRunCluster(t, 1)
-		defer c.Close()
-		hldr := test.Holder{Holder: c[0].Server.Holder()}
-		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
-		f1, err := index.CreateField("f1", pilosa.OptFieldTypeDefault())
-		if err != nil {
-			t.Fatal(err)
-		}
-		f2, err := index.CreateField("f2", pilosa.OptFieldTypeDefault())
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		q := fmt.Sprintf(`Store(Distinct(field=%s), %s=2)`, f1.Name(), f2.Name())
-		if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: index.Name(), Query: q}); err == nil {
-			t.Fatalf("expected 'unsupported result type' error, got: %+v", res)
-		}
-	})
 }
 
 func benchmarkExistence(nn bool, b *testing.B) {
@@ -5312,6 +5295,7 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt
 	defer c.Close()
 	hldr := test.Holder{Holder: c[0].Server.Holder()}
 	index := hldr.MustCreateIndexIfNotExists("i", *indexOptions)
+	defer index.Close()
 	_, err := index.CreateField("f", fieldOption...)
 	if err != nil {
 		t.Fatal(err)
@@ -5723,12 +5707,15 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
 	if err := json.NewDecoder(bytes.NewReader(data)).Decode(schema); err != nil {
 		t.Fatal(err)
 	}
+
 	if err := api.ApplySchema(context.TODO(), schema, false); err != nil {
 		t.Fatal(err)
 	}
 
+	// AntitodePoint == row 1 b/c keys field.
 	writeQuery := `Set(100, type=AntidotePoint)Set(100, equip_id=100)Set(100, site_id=100)Set(100, id=100)`
-	for _, i := range schema.Indexes {
+	for k, i := range schema.Indexes {
+		_ = k
 		if _, err := api.Query(context.TODO(), &pilosa.QueryRequest{Index: i.Name, Query: writeQuery}); err != nil {
 			t.Fatal(err)
 		}
@@ -5739,11 +5726,11 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
 		Intersect(
 			Distinct(
 				Intersect(Row(type=AntidotePoint)),
-			index=power_ts, field=equip_id),
+			index=equipment, field=equip_id),
 			Distinct(
 				Intersect(Row(type=AntidotePoint)),
-			index=power_ts, field=equip_id)
-		), index=equipment, field=site_id)`
+			index=sites, field=equip_id)
+		), index=power_ts, field=site_id)`
 
 	// Check if test query gives correct results (one column 100)
 	t.Run("Distinct", func(t *testing.T) {
@@ -5760,6 +5747,7 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
 		}
 		if r.Pos.Count() != 1 {
 			t.Fatalf("invalid pilosa.SignedRow.Pos.Count, expected: 1, got: %v", r.Pos.Count())
+
 		}
 		if r.Pos.Columns()[0] != 100 {
 			t.Fatalf("invalid pilosa.SignedRow.Pos.Columns, expected: [100], got: %v", r.Pos.Columns())
diff --git a/extension.go b/extension.go
deleted file mode 100644
index 1a492cce3..000000000
--- a/extension.go
+++ /dev/null
@@ -1,96 +0,0 @@
-// Copyright 2019 Pilosa Corp.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pilosa
-
-import (
-	"fmt"
-
-	"github.com/molecula/ext"
-	"github.com/pilosa/pilosa/v2/roaring"
-)
-
-// WrapBitmap yields an extension-Bitmap from a roaring Bitmap.
-func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap {
-	return wrappedBitmap{bm}
-}
-
-// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to
-// an extension Bitmap.
-type wrappedBitmap struct{ *roaring.Bitmap }
-
-// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
-func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap {
-	if inner, ok := bm.(wrappedBitmap); ok {
-		if inner.Bitmap != nil {
-			return inner.Bitmap
-		}
-		return roaring.NewFileBitmap()
-	}
-	return roaring.NewFileBitmap()
-}
-
-func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap {
-	return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)}
-}
-
-func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap {
-	return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)}
-}
-
-func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 {
-	return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap)
-}
-
-func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap {
-	return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)}
-}
-
-func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap {
-	return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)}
-}
-
-func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) {
-	shifted, err := b.Bitmap.Shift(n)
-	return wrappedBitmap{shifted}, err
-}
-
-func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap {
-	return wrappedBitmap{b.Bitmap.Flip(start, last)}
-}
-
-func (b wrappedBitmap) New() ext.Bitmap {
-	return WrapBitmap(roaring.NewFileBitmap())
-}
-
-// ContainerBits tries to get one container's worth of bits.
-func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) {
-	// it's an error to call this with a non-container-aligned offset
-	if offset&0xFFFF != 0 {
-		return nil
-	}
-	if b.Bitmap == nil {
-		fmt.Printf("ContainerBits on bitmap with no contents\n")
-		return nil
-	}
-	if b.Bitmap.Containers == nil {
-		fmt.Printf("ContainerBits on bitmap with nil Containers\n")
-		return nil
-	}
-	c := b.Bitmap.Containers.Get(offset >> 16)
-	if c == nil {
-		return nil
-	}
-	return c.AsBitmap(target)
-}
diff --git a/field.go b/field.go
index a9a80e2fe..0802c3848 100644
--- a/field.go
+++ b/field.go
@@ -92,6 +92,8 @@ type Field struct {
 	name          string
 	qualifiedName string
 
+	idx *Index
+
 	viewMap map[string]*view
 
 	// Row attribute storage and cache
@@ -1181,6 +1183,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) {
 
 func (f *Field) newView(path, name string) *view {
 	view := newView(f.holder, path, f.index, f.name, name, f.options)
+	view.idx = f.idx
 	view.rowAttrStore = f.rowAttrStore
 	view.stats = f.Stats
 	view.broadcaster = f.broadcaster
@@ -1432,6 +1435,14 @@ func (f *Field) SetValue(tx Tx, columnID uint64, value int64) (changed bool, err
 	if err != nil {
 		return false, errors.Wrap(err, "creating view")
 	}
+	if view.holder == nil {
+		panic("view.holder should not be nil")
+	}
+	if view.idx == nil {
+		panic("view.idx should not be nil")
+	}
+	view.holder.addIndexFromField(view.idx)
+
 	return view.setValue(tx, columnID, bsig.BitDepth, baseValue)
 }
 
@@ -1761,6 +1772,10 @@ func (f *Field) importRoaring(ctx context.Context, tx Tx, data []byte, shard uin
 	return nil
 }
 
+func (f *Field) GetIndex() *Index {
+	return f.idx
+}
+
 func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, block int) error {
 	span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaringOverwrite")
 	defer span.Finish()
@@ -1787,7 +1802,7 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte,
 	switch f.Options().Type {
 	case FieldTypeInt, FieldTypeDecimal:
 		frag.mu.Lock()
-		if err := frag.calculateMaxRowID(); err != nil {
+		if err := frag.calculateMaxRowID(tx); err != nil {
 			return err
 		}
 		maxRowID, _, err := frag.maxRow(tx, nil)
@@ -2132,6 +2147,9 @@ func isValidCacheType(v string) bool {
 	}
 }
 
+// TODO(jea): why isn't this bits.Len64(x) using import "math/bits"
+// That would be much (80x or more) faster and correct if the high bit is set.
+//
 // bitDepth returns the number of bits required to store a value.
 func bitDepth(v uint64) uint {
 	for i := uint(0); i < 63; i++ {
diff --git a/field_internal_test.go b/field_internal_test.go
index 8a5aaffe4..529cabee8 100644
--- a/field_internal_test.go
+++ b/field_internal_test.go
@@ -205,10 +205,17 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
 	if err != nil {
 		t.Fatal(err)
 	}
-	field, err := NewField(NewHolder(DefaultPartitionN), path, "i", "f", opts)
+	h := NewHolder(DefaultPartitionN)
+	h.Path = path
+	idx, err := h.CreateIndex("i", IndexOptions{})
+	if err != nil {
+		panic(err)
+	}
+	field, err := NewField(h, path, "i", "f", opts)
 	if err != nil {
 		t.Fatal(err)
 	}
+	field.idx = idx
 	return &TestField{Field: field}
 }
 
@@ -223,6 +230,9 @@ func OpenField(t *testing.T, opts FieldOption) *TestField {
 
 // Close closes the field and removes the underlying data.
 func (f *TestField) Close() error {
+	if f.idx != nil {
+		panicOn(f.idx.Txf.CloseIndex(f.idx))
+	}
 	defer os.RemoveAll(f.Path())
 	return f.Field.Close()
 }
@@ -235,10 +245,17 @@ func (f *TestField) Reopen() error {
 	}
 
 	path, index, name := f.Path(), f.Index(), f.Name()
-	f.Field, err = NewField(NewHolder(DefaultPartitionN), path, index, name, OptFieldTypeDefault())
+	h := NewHolder(DefaultPartitionN)
+	h.Path = path
+	idx, err := h.CreateIndex(index, IndexOptions{})
 	if err != nil {
 		return err
 	}
+	f.Field, err = NewField(h, path, index, name, OptFieldTypeDefault())
+	if err != nil {
+		return err
+	}
+	f.Field.idx = idx
 
 	if err := f.Open(); err != nil {
 		return err
@@ -311,7 +328,8 @@ func TestField_RowTime(t *testing.T) {
 	defer f.Close()
 
 	// Obtain transaction.
-	tx := &RoaringTx{Field: f.Field}
+	tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
+	defer tx.Rollback()
 
 	if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
 		t.Fatal(err)
@@ -323,6 +341,12 @@ func TestField_RowTime(t *testing.T) {
 	f.MustSetBit(tx, 1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC))
 	f.MustSetBit(tx, 1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC))
 
+	panicOn(tx.Commit())
+
+	// obtain 2nd transaction to read it back.
+	tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
+	defer tx.Rollback()
+
 	if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
 		t.Fatal(err)
 	} else if !reflect.DeepEqual(r.Columns(), []uint64{1, 3, 4, 5}) {
@@ -358,6 +382,7 @@ func TestField_RowTime(t *testing.T) {
 func TestField_PersistAvailableShards(t *testing.T) {
 	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
 	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap(1, 2, 3)
@@ -379,6 +404,7 @@ func TestField_PersistAvailableShards(t *testing.T) {
 func TestField_CorruptAvailableShards(t *testing.T) {
 	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
 	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap(1, 2, 3)
@@ -411,6 +437,7 @@ func TestField_CorruptAvailableShards(t *testing.T) {
 func TestField_TruncatedAvailableShards(t *testing.T) {
 	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
 	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap(1, 2, 3)
@@ -441,6 +468,7 @@ func TestField_TruncatedAvailableShards(t *testing.T) {
 func TestField_PersistAvailableShardsFootprint(t *testing.T) {
 	availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
 	f := OpenField(t, OptFieldTypeDefault())
+	defer f.Close()
 
 	// bm represents remote available shards.
 	bm := roaring.NewBitmap()
@@ -554,6 +582,7 @@ func TestField_ApplyOptions(t *testing.T) {
 // to result in a value of 9 instead of 1.
 func TestBSIGroup_importValue(t *testing.T) {
 	f := OpenField(t, OptFieldTypeInt(-100, 200))
+	defer f.Close()
 
 	options := &ImportOptions{}
 	for i, tt := range []struct {
@@ -581,12 +610,17 @@ func TestBSIGroup_importValue(t *testing.T) {
 			[]uint64{100},
 		},
 	} {
-		tx := &RoaringTx{Field: f.Field}
+		tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
+		defer tx.Rollback()
 
 		if err := f.importValue(tx, tt.columnIDs, tt.values, options); err != nil {
 			t.Fatalf("test %d, importing values: %s", i, err.Error())
 		}
 
+		panicOn(tx.Commit())
+		tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
+		defer tx.Rollback()
+
 		if row, err := f.Range(tx, f.name, pql.EQ, tt.checkVal); err != nil {
 			t.Fatalf("test %d, getting range: %s", i, err.Error())
 		} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
@@ -597,6 +631,7 @@ func TestBSIGroup_importValue(t *testing.T) {
 
 func TestIntField_MinMaxForShard(t *testing.T) {
 	f := OpenField(t, OptFieldTypeInt(-100, 200))
+	defer f.Close()
 
 	options := &ImportOptions{}
 	for i, test := range []struct {
@@ -648,12 +683,17 @@ func TestIntField_MinMaxForShard(t *testing.T) {
 		},
 	} {
 		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
-			tx := &RoaringTx{Field: f.Field}
+			tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
+			defer tx.Rollback()
 
 			if err := f.importValue(tx, test.columnIDs, test.values, options); err != nil {
 				t.Fatalf("test %d, importing values: %s", i, err.Error())
 			}
 
+			panicOn(tx.Commit())
+			tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
+			defer tx.Rollback()
+
 			maxvc, err := f.MaxForShard(tx, 0, nil)
 			if err != nil {
 				t.Fatalf("getting max for shard: %v", err)
@@ -753,6 +793,7 @@ func TestDecimalField_MinMaxBoundaries(t *testing.T) {
 
 func TestDecimalField_MinMaxForShard(t *testing.T) {
 	f := OpenField(t, OptFieldTypeDecimal(3))
+	defer f.Close()
 
 	options := &ImportOptions{}
 	for i, test := range []struct {
@@ -804,12 +845,17 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
 		},
 	} {
 		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
-			tx := &RoaringTx{Field: f.Field}
+			tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
+			defer tx.Rollback()
 
 			if err := f.importFloatValue(tx, test.columnIDs, test.values, options); err != nil {
 				t.Fatalf("test %d, importing values: %s", i, err.Error())
 			}
 
+			panicOn(tx.Commit())
+			tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
+			defer tx.Rollback()
+
 			maxvc, err := f.MaxForShard(tx, 0, nil)
 			if err != nil {
 				t.Fatalf("getting max for shard: %v", err)
diff --git a/field_test.go b/field_test.go
index aeade9fcd..2fc804b7c 100644
--- a/field_test.go
+++ b/field_test.go
@@ -25,6 +25,8 @@ import (
 	"github.com/pilosa/pilosa/v2/test"
 )
 
+var panicOn = pilosa.PanicOn
+
 // Ensure a field can set & read a bsiGroup value.
 func TestField_SetValue(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
@@ -35,7 +37,10 @@ func TestField_SetValue(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		tx := &pilosa.RoaringTx{Field: f.Field}
+
+		idxPilosa := f.Field.GetIndex()
+		tx := idxPilosa.NewTx(pilosa.Txo{Write: writable, Index: idxPilosa, Field: f.Field})
+		defer tx.Rollback()
 
 		// Set value on field.
 		if changed, err := f.SetValue(tx, 100, 21); err != nil {
@@ -69,7 +74,9 @@ func TestField_SetValue(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		tx := &pilosa.RoaringTx{Field: f.Field}
+		idxP := f.Field.GetIndex()
+		tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
+		defer tx.Rollback()
 
 		// Set value.
 		if changed, err := f.SetValue(tx, 100, 21); err != nil {
@@ -103,7 +110,9 @@ func TestField_SetValue(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		tx := &pilosa.RoaringTx{Field: f.Field}
+		idxP := f.Field.GetIndex()
+		tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
+		defer tx.Rollback()
 
 		// Set value.
 		if _, err := f.SetValue(tx, 100, 21); err != pilosa.ErrBSIGroupNotFound {
@@ -119,7 +128,9 @@ func TestField_SetValue(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		tx := &pilosa.RoaringTx{Field: f.Field}
+		idxP := f.Field.GetIndex()
+		tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
+		defer tx.Rollback()
 
 		// Set value.
 		if _, err := f.SetValue(tx, 100, 15); err != pilosa.ErrBSIGroupValueTooLow {
@@ -135,7 +146,9 @@ func TestField_SetValue(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		tx := &pilosa.RoaringTx{Field: f.Field}
+		idxP := f.Field.GetIndex()
+		tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
+		defer tx.Rollback()
 
 		// Set value.
 		if _, err := f.SetValue(tx, 100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
@@ -204,7 +217,9 @@ func TestField_AvailableShards(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	tx := &pilosa.RoaringTx{Field: f.Field}
+	idxP := f.Field.GetIndex()
+	tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
+	defer tx.Rollback()
 
 	// Set values on shards 0 & 2, and verify.
 	if _, err := f.SetBit(tx, 0, 100, nil); err != nil {
@@ -214,6 +229,7 @@ func TestField_AvailableShards(t *testing.T) {
 	} else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
 		t.Fatal(diff)
 	}
+	panicOn(tx.Commit())
 
 	// Set remote shards and verify.
 	if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil {
@@ -244,7 +260,9 @@ func TestField_ClearValue(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		tx := &pilosa.RoaringTx{Field: f.Field}
+		idxP := f.Field.GetIndex()
+		tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
+		defer tx.Rollback()
 
 		// Set value on field.
 		if changed, err := f.SetValue(tx, 100, 21); err != nil {
@@ -252,6 +270,9 @@ func TestField_ClearValue(t *testing.T) {
 		} else if !changed {
 			t.Fatal("expected change")
 		}
+		panicOn(tx.Commit())
+
+		tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field})
 
 		// Read value.
 		if value, exists, err := f.Value(tx, 100); err != nil {
@@ -261,12 +282,17 @@ func TestField_ClearValue(t *testing.T) {
 		} else if !exists {
 			t.Fatal("expected value to exist")
 		}
+		tx.Rollback()
+		tx = idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
 
 		if changed, err := f.ClearValue(tx, 100); err != nil {
 			t.Fatal(err)
 		} else if !changed {
 			t.Fatal(err)
 		}
+		panicOn(tx.Commit())
+		tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field})
+		defer tx.Rollback()
 
 		// Read value.
 		if _, exists, err := f.Value(tx, 100); err != nil {
diff --git a/fragment.go b/fragment.go
index 9734429ed..9dc5b205b 100644
--- a/fragment.go
+++ b/fragment.go
@@ -108,6 +108,9 @@ type fragment struct {
 	view  string
 	shard uint64
 
+	// idx cached to avoid repeatedly looking it up everywhere.
+	idx *Index
+
 	// parent holder, used to find snapshot queue, etc.
 	holder *Holder
 
@@ -180,6 +183,10 @@ type fragment struct {
 
 // newFragment returns a new instance of Fragment.
 func newFragment(holder *Holder, path, index, field, view string, shard uint64, flags byte) *fragment {
+	idx := holder.Index(index)
+	if idx == nil {
+		panic(fmt.Sprintf("holder=%#v but got nil idx back from holder!", holder))
+	}
 	f := &fragment{
 		path:  path,
 		index: index,
@@ -187,6 +194,7 @@ func newFragment(holder *Holder, path, index, field, view string, shard uint64,
 		view:  view,
 		shard: shard,
 		flags: flags,
+		idx:   idx,
 
 		CacheType: DefaultCacheType,
 		CacheSize: DefaultCacheSize,
@@ -250,7 +258,9 @@ func (f *fragment) Open() error {
 		f.checksums = make(map[int][]byte)
 
 		// Read last bit to determine max row.
-		return f.calculateMaxRowID()
+		tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f})
+		defer tx.Rollback()
+		return f.calculateMaxRowID(tx)
 	}(); err != nil {
 		f.close()
 		return err
@@ -398,6 +408,15 @@ func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation,
 // logic is now mostly in importStorage (reading in a bitmap) and applyStorage
 // (remapping an existing bitmap to match a new backing store).
 func (f *fragment) openStorage(unmarshalData bool) error {
+
+	if !f.idx.NeedsSnapshot() {
+		f.gen = &NopGeneration{}
+		f.rowCache = &simpleCache{make(map[uint64]*Row)}
+		f.currdata = struct{ from, to uintptr }{}
+		f.prevdata = f.currdata
+		return nil // openStorage becomes a noop under RBF, Badger, etc.
+	}
+
 	// Create a roaring bitmap to serve as storage for the shard.
 	if f.storage == nil {
 		f.storage = roaring.NewFileBitmap()
@@ -473,10 +492,16 @@ func (f *fragment) openCache() error {
 		return nil
 	}
 
+	tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f})
+	defer tx.Rollback()
+
 	// Read in all rows by ID.
 	// This will cause them to be added to the cache.
 	for _, id := range pb.IDs {
-		n := f.storage.CountRange(id*ShardWidth, (id+1)*ShardWidth)
+		n, err := tx.CountRange(f.index, f.field, f.view, f.shard, id*ShardWidth, (id+1)*ShardWidth)
+		if err != nil {
+			return errors.Wrap(err, "CountRange")
+		}
 		f.cache.BulkAdd(id, n)
 	}
 	f.cache.Invalidate()
@@ -583,7 +608,7 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) {
 
 	row := &Row{
 		segments: []rowSegment{{
-			data:     data, // this data contains BadgerTx data, which should not survive Txn commit.
+			data:     data,
 			shard:    f.shard,
 			writable: true,
 		}},
@@ -598,7 +623,11 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) {
 func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err = f.gen.Transaction(wp, func() error {
 		// handle mutux field type
 		if f.mutexVector != nil {
 			if err := f.handleMutex(tx, rowID, columnID); err != nil {
@@ -680,7 +709,11 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
 func (f *fragment) clearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err = f.gen.Transaction(wp, func() error {
 		changed, err = f.unprotectedClearBit(tx, rowID, columnID)
 		return err
 	})
@@ -739,7 +772,11 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b
 func (f *fragment) setRow(tx Tx, row *Row, rowID uint64) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err = f.gen.Transaction(wp, func() error {
 		changed, err = f.unprotectedSetRow(tx, row, rowID)
 		return err
 	})
@@ -802,7 +839,11 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo
 func (f *fragment) clearRow(tx Tx, rowID uint64) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err = f.gen.Transaction(wp, func() error {
 		changed, err = f.unprotectedClearRow(tx, rowID)
 		return err
 	})
@@ -845,7 +886,11 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e
 // This updates both the on-disk storage and the in-cache bitmap.
 func (f *fragment) unprotectedClearBlock(tx Tx, block int) (changed bool, err error) {
 	firstRow := uint64(block * HashBlockSize)
-	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err = f.gen.Transaction(wp, func() error {
 		var rowChanged bool
 		for rowID := uint64(firstRow); rowID < firstRow+HashBlockSize; rowID++ {
 			if changed, err := f.unprotectedClearRow(tx, rowID); err != nil {
@@ -953,8 +998,11 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
 func (f *fragment) setValueBase(tx Tx, columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-
-	err = f.gen.Transaction(&f.storage.OpWriter, func() error {
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err = f.gen.Transaction(wp, func() error {
 		// Convert value to an unsigned representation.
 		uvalue := uint64(value)
 		if value < 0 {
@@ -1291,8 +1339,12 @@ func (f *fragment) maxRow(tx Tx, filter *Row) (uint64, uint64, error) {
 
 // calculateMaxRowID determines the field's maxRowID value based
 // on the contents of its storage, and sets the struct argument.
-func (f *fragment) calculateMaxRowID() (err error) {
-	f.maxRowID = f.storage.Max() / ShardWidth
+func (f *fragment) calculateMaxRowID(tx Tx) (err error) {
+	max, err := tx.Max(f.index, f.field, f.view, f.shard)
+	if err != nil {
+		return err
+	}
+	f.maxRowID = max / ShardWidth
 	return nil
 }
 
@@ -1484,7 +1536,6 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality
 	if err != nil {
 		return nil, err
 	}
-
 	// Create predicate without sign bit.
 	upredicate := absInt64(predicate)
 
@@ -1492,7 +1543,6 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality
 	if err != nil {
 		return nil, err
 	}
-
 	switch {
 	case predicate == 0 && !allowEquality:
 		// Match all positive numbers except zero.
@@ -2217,7 +2267,11 @@ func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options
 // operations to the op log.
 func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error {
 	//tx.AddN()
-	err := f.gen.Transaction(&f.storage.OpWriter, func() error {
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err := f.gen.Transaction(wp, func() error { // segfault
 		if len(set) > 0 {
 			f.stats.Count(MetricImportingN, int64(len(set)), 1)
 
@@ -2453,14 +2507,18 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b
 	span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits")
 	var changed int
 	var rowSet map[uint64]int
-	err := f.gen.Transaction(&f.storage.OpWriter, func() (err error) {
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err := f.gen.Transaction(wp, func() (err error) {
 		var rit roaring.RoaringIterator
 		rit, err = roaring.NewRoaringIterator(data)
 		if err != nil {
 			return err
 		}
 
-		changed, rowSet, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, rit, clear, true, rowSize)
+		changed, rowSet, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, rit, clear, true, rowSize, nil)
 		return err
 	})
 
@@ -2550,6 +2608,9 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg
 // snapshot does the actual snapshot operation. it does not check or care
 // about f.snapshotPending.
 func (f *fragment) snapshot() (err error) {
+	if !f.idx.NeedsSnapshot() {
+		return nil
+	}
 	if !f.open {
 		return errors.New("snapshot request on closed fragment")
 	}
@@ -2688,31 +2749,16 @@ func (f *fragment) WriteTo(w io.Writer) (n int64, err error) {
 	return 0, nil
 }
 
+// used in shipping the slices across the network for a resize.
 func (f *fragment) writeStorageToArchive(tw *tar.Writer) error {
-	// Open separate file descriptor to read from.
-	file, err := os.Open(f.path)
+
+	tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx})
+	defer tx.Rollback()
+	file, sz, err := tx.RoaringBitmapReader(f.index, f.field, f.view, f.shard, f.path)
 	if err != nil {
-		return errors.Wrap(err, "opening file")
-	}
-	defer file.Close()
-
-	// Retrieve the current file size under lock so we don't read
-	// while an operation is appending to the end.
-	var sz int64
-	if err := func() error {
-		f.mu.Lock()
-		defer f.mu.Unlock()
-
-		fi, err := file.Stat()
-		if err != nil {
-			return errors.Wrap(err, "statting")
-		}
-		sz = fi.Size()
-
-		return nil
-	}(); err != nil {
 		return err
 	}
+	defer file.Close()
 
 	// Write archive header.
 	if err := tw.WriteHeader(&tar.Header{
@@ -2779,9 +2825,15 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) {
 		// Process file based on file name.
 		switch hdr.Name {
 		case "data":
-			if err := f.readStorageFromArchive(tr); err != nil {
+			idx := f.holder.Index(f.index)
+			tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+			if err := f.fillFragmentFromArchive(tx, tr); err != nil {
 				return 0, errors.Wrap(err, "reading storage")
 			}
+			if err := tx.Commit(); err != nil {
+				return 0, errors.Wrap(err, "Commit after tx.ReadFragmentFromArchive")
+			}
 		case "cache":
 			if err := f.readCacheFromArchive(tr); err != nil {
 				return 0, errors.Wrap(err, "reading cache")
@@ -2794,7 +2846,40 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) {
 	return 0, nil
 }
 
+// should be morally equivalent to fragment.readStorageFromArchive()
+// below for RoaringTx, but also work on any Tx because it uses
+// tx.ImportRoaringBits().
+func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error {
+
+	// this is reading from inside a tarball, so definitely no need
+	// to close it here.
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return errors.Wrap(err, "fillFragmentFromArchive ioutil.ReadAll(r)")
+	}
+	if len(data) == 0 {
+		return nil
+	}
+
+	// For reference, compare to what fragment.go:313 fragment.importStorage() does.
+
+	clear := false
+	log := false
+	rowSize := uint64(0)
+	itr, err := roaring.NewRoaringIterator(data)
+	if err != nil {
+		return errors.Wrap(err, "fillFragmentFromArchive NewRoaringIterator")
+	}
+	changed, rowSet, err := tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, itr, clear, log, rowSize, data)
+	_, _ = changed, rowSet
+	if err != nil {
+		return errors.Wrap(err, "fillFragmentFromArchive ImportRoaringBits")
+	}
+	return nil
+}
+
 func (f *fragment) readStorageFromArchive(r io.Reader) error {
+
 	// Create a temporary file to copy into.
 	path := f.path + copyExt
 	file, err := os.Create(path)
@@ -2808,6 +2893,12 @@ func (f *fragment) readStorageFromArchive(r io.Reader) error {
 		return errors.Wrap(err, "copying")
 	}
 
+	// TODO(jea): isn't this next Rename a file handle leak?
+	// try closing first
+	if err := f.closeStorage(); err != nil {
+		return errors.Wrap(err, "closeStorage-prior-to-Rename-and-openStorage")
+	}
+
 	// Move snapshot to data file location.
 	if err := os.Rename(path, f.path); err != nil {
 		return errors.Wrap(err, "renaming")
diff --git a/fragment_internal_test.go b/fragment_internal_test.go
index dca084b5a..ff0de833f 100644
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@@ -24,6 +24,7 @@ import (
 	"math"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"reflect"
 	"runtime"
 	"runtime/debug"
@@ -445,7 +446,7 @@ func TestFragment_SetValue(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		// Set value.
 		if changed, err := f.setValue(tx, 100, 10, 20); err != nil {
@@ -481,7 +482,7 @@ func TestFragment_SetValue(t *testing.T) {
 
 			// Obtain transaction.
 			tx := f.txTestingOnly
-			defer tx.Rollback() //LOOKATME
+			defer tx.Rollback()
 
 			// Set values.
 			m := make(map[uint64]int64)
@@ -538,12 +539,11 @@ func TestFragment_Sum(t *testing.T) {
 	const bitDepth = 16
 
 	f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
-	_ = idx
 	defer f.Clean(t)
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set values.
 	vals := []struct {
@@ -562,6 +562,10 @@ func TestFragment_Sum(t *testing.T) {
 		}
 	}
 
+	panicOn(tx.Commit())
+	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+	defer tx.Rollback()
+
 	t.Run("NoFilter", func(t *testing.T) {
 		if sum, n, err := f.sum(tx, nil, bitDepth); err != nil {
 			t.Fatal(err)
@@ -582,10 +586,19 @@ func TestFragment_Sum(t *testing.T) {
 		}
 	})
 
+	panicOn(tx.Commit())
+	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
+	defer tx.Rollback()
+
 	// verify that clearValue clears values
 	if _, err := f.clearValue(tx, 1000, bitDepth, 23); err != nil {
 		t.Fatal(err)
 	}
+
+	panicOn(tx.Commit())
+	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+	defer tx.Rollback()
+
 	t.Run("ClearValue", func(t *testing.T) {
 		if sum, n, err := f.sum(tx, nil, bitDepth); err != nil {
 			t.Fatal(err)
@@ -602,12 +615,11 @@ func TestFragment_MinMax(t *testing.T) {
 	const bitDepth = 16
 
 	f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
-	_ = idx
 	defer f.Clean(t)
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set values.
 	if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil {
@@ -626,6 +638,12 @@ func TestFragment_MinMax(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	panicOn(tx.Commit())
+
+	// the new tx is shared by Min/Max below.
+	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+	defer tx.Rollback()
+
 	t.Run("Min", func(t *testing.T) {
 		tests := []struct {
 			filter *Row
@@ -689,7 +707,7 @@ func TestFragment_Range(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		// Set values.
 		if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil {
@@ -716,7 +734,8 @@ func TestFragment_Range(t *testing.T) {
 		defer f.Clean(t)
 
 		// Obtain transaction.
-		tx := &RoaringTx{fragment: f}
+		tx := f.txTestingOnly
+		defer tx.Rollback()
 
 		// Set values.
 		if _, err := f.setValue(tx, 1000, 1, 0); err != nil {
@@ -745,7 +764,7 @@ func TestFragment_Range(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		// Set values.
 		if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil {
@@ -773,7 +792,7 @@ func TestFragment_Range(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		// Set values.
 		if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil {
@@ -826,7 +845,7 @@ func TestFragment_Range(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		if _, err := f.setValue(tx, 1, 1, 1); err != nil {
 			t.Fatal(err)
@@ -846,7 +865,7 @@ func TestFragment_Range(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		if _, err := f.setValue(tx, 1, 2, 3); err != nil {
 			t.Fatal(err)
@@ -868,7 +887,7 @@ func TestFragment_Range(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		// Set values.
 		if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil {
@@ -921,7 +940,7 @@ func TestFragment_Range(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		if _, err := f.setValue(tx, 1, 2, 0); err != nil {
 			t.Fatal(err)
@@ -942,7 +961,8 @@ func TestFragment_Range(t *testing.T) {
 		defer f.Clean(t)
 
 		// Obtain transaction.
-		tx := &RoaringTx{fragment: f}
+		tx := f.txTestingOnly
+		defer tx.Rollback()
 
 		if _, err := f.setValue(tx, 1, 2, 0); err != nil {
 			t.Fatal(err)
@@ -964,7 +984,7 @@ func TestFragment_Range(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		// Set values.
 		if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil {
@@ -1016,7 +1036,8 @@ func TestFragment_Range(t *testing.T) {
 		defer f.Clean(t)
 
 		// Obtain transaction.
-		tx := &RoaringTx{fragment: f}
+		tx := f.txTestingOnly
+		defer tx.Rollback()
 
 		if _, err := f.setValue(tx, 1, 64, 0xf0); err != nil {
 			t.Fatal(err)
@@ -1037,7 +1058,7 @@ func TestFragment_Range(t *testing.T) {
 func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) {
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	column := uint64(0)
 	for i := 0; i < b.N; i++ {
@@ -1074,7 +1095,7 @@ func BenchmarkFragment_SetValue(b *testing.B) {
 func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) {
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	column := uint64(0)
 	b.StopTimer()
@@ -1145,7 +1166,7 @@ func BenchmarkFragment_RepeatedSmallImports(b *testing.B) {
 
 							// Obtain transaction.
 							tx := f.txTestingOnly
-							defer tx.Rollback() //LOOKATME
+							defer tx.Rollback()
 
 							err := f.importRoaringT(tx, getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false)
 							if err != nil {
@@ -1162,6 +1183,7 @@ func BenchmarkFragment_RepeatedSmallImports(b *testing.B) {
 									b.Fatalf("doing small bulk import: %v", err)
 								}
 							}
+							tx.Rollback() // don't exhaust the Tx space under b.N iterations.
 						}
 					})
 				}
@@ -1239,7 +1261,7 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) {
 
 						// Obtain transaction.
 						tx := f.txTestingOnly
-						defer tx.Rollback() //LOOKATME
+						defer tx.Rollback()
 
 						err := f.importValue(tx, initialCols, initialVals, 21, false)
 						if err != nil {
@@ -1257,6 +1279,7 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) {
 								b.Fatalf("importing values: %v", err)
 							}
 						}
+						tx.Rollback() // don't exhaust the Tx over the b.N iterations.
 					}
 				})
 			}
@@ -1268,12 +1291,11 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) {
 // Ensure a fragment can snapshot correctly.
 func TestFragment_Snapshot(t *testing.T) {
 	f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
-	_ = idx
 	defer f.Clean(t)
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set and then clear bits on the fragment.
 	if _, err := f.setBit(tx, 1000, 1); err != nil {
@@ -1283,6 +1305,9 @@ func TestFragment_Snapshot(t *testing.T) {
 	} else if _, err := f.clearBit(tx, 1000, 1); err != nil {
 		t.Fatal(err)
 	}
+	panicOn(tx.Commit())
+	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+	defer tx.Rollback()
 
 	// Snapshot bitmap and verify data.
 	if err := f.Snapshot(); err != nil {
@@ -1307,7 +1332,7 @@ func TestFragment_ForEachBit(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set bits on the fragment.
 	if _, err := f.setBit(tx, 100, 20); err != nil {
@@ -1341,7 +1366,7 @@ func TestFragment_Top(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set bits on the rows 100, 101, & 102.
 	f.mustSetBits(tx, 100, 1, 3, 200)
@@ -1364,12 +1389,11 @@ func TestFragment_Top(t *testing.T) {
 // Ensure a fragment can filter rows when retrieving the top n rows.
 func TestFragment_Top_Filter(t *testing.T) {
 	f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked)
-	_ = idx
 	defer f.Clean(t)
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set bits on the rows 100, 101, & 102.
 	f.mustSetBits(tx, 100, 1, 3, 200)
@@ -1386,6 +1410,10 @@ func TestFragment_Top_Filter(t *testing.T) {
 		t.Fatalf("setAttrs: %v", err)
 	}
 
+	panicOn(tx.Commit())
+	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+	defer tx.Rollback()
+
 	// Retrieve top rows.
 	if pairs, err := f.top(tx, topOptions{
 		N:            2,
@@ -1410,7 +1438,7 @@ func TestFragment_TopN_Intersect(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Create an intersecting input row.
 	src := NewRow(1, 2, 3)
@@ -1446,7 +1474,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Create an intersecting input row.
 	src := NewRow(
@@ -1499,7 +1527,7 @@ func TestFragment_TopN_IDs(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set bits on various rows.
 	f.mustSetBits(tx, 100, 1, 2, 3)
@@ -1525,7 +1553,7 @@ func TestFragment_TopN_NopCache(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set bits on various rows.
 	f.mustSetBits(tx, 100, 1, 2, 3)
@@ -1700,7 +1728,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
 	defer f.Clean(t)
 
 	// Obtain transaction.
-	tx := f.txTestingOnly //&RoaringTx{fragment: f}
+	tx := f.txTestingOnly
 	defer tx.Rollback()
 
 	// Set bits on a different block.
@@ -1727,7 +1755,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set bits on the fragment.
 	for i := uint64(0); i < 1000; i++ {
@@ -1781,6 +1809,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
 
 	// Obtain transaction.
 	tx := index.Txf.NewTx(Txo{Write: writable, Index: index, Fragment: f})
+	defer tx.Rollback()
 
 	// Set bits on the fragment.
 	for i := uint64(0); i < 1000; i++ {
@@ -1789,6 +1818,10 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
 		}
 	}
 
+	panicOn(tx.Commit())
+	tx = index.Txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f})
+	defer tx.Rollback()
+
 	// Verify correct cache type and size.
 	if cache, ok := f.cache.(*rankCache); !ok {
 		t.Fatalf("unexpected cache: %T", f.cache)
@@ -1819,7 +1852,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
 	defer f0.Clean(t)
 
 	// Obtain transaction.
-	tx := &RoaringTx{fragment: f0}
+	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f0})
+	defer tx.Rollback()
 
 	// Set and then clear bits on the fragment.
 	if _, err := f0.setBit(tx, 1000, 1); err != nil {
@@ -1847,7 +1881,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
 	_ = idx
 	defer f1.Clean(t)
 
-	if rn, err := f1.ReadFrom(&buf); err != nil {
+	if rn, err := f1.ReadFrom(&buf); err != nil { // eventually calls fragment.fillFragmentFromArchive
 		t.Fatal(err)
 	} else if wn != rn {
 		t.Fatalf("read/write byte count mismatch: wn=%d, rn=%d", wn, rn)
@@ -1898,13 +1932,12 @@ func BenchmarkFragment_Blocks(b *testing.B) {
 
 func BenchmarkFragment_IntersectionCount(b *testing.B) {
 	f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
-	_ = idx
 	defer f.Clean(b)
 	f.MaxOpN = math.MaxInt32
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Generate some intersecting data.
 	for i := 0; i < 10000; i += 2 {
@@ -1918,6 +1951,10 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
 		}
 	}
 
+	panicOn(tx.Commit())
+	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+	defer tx.Rollback()
+
 	// Snapshot to disk before benchmarking.
 	if err := f.Snapshot(); err != nil {
 		b.Fatal(err)
@@ -1939,7 +1976,7 @@ func TestFragment_Tanimoto(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	src := NewRow(1, 2, 3)
 
@@ -1967,7 +2004,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	src := NewRow(1, 2, 3)
 
@@ -1997,7 +2034,7 @@ func TestFragment_Snapshot_Run(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// Set bits on the fragment.
 	for i := uint64(1); i < 3; i++ {
@@ -2029,7 +2066,7 @@ func TestFragment_SetMutex(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	var cols []uint64
 
@@ -2148,7 +2185,7 @@ func TestFragment_ImportSet(t *testing.T) {
 
 			// Obtain transaction.
 			tx := f.txTestingOnly
-			defer tx.Rollback() //LOOKATME
+			defer tx.Rollback()
 
 			// Set import.
 			err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{})
@@ -2181,6 +2218,140 @@ func TestFragment_ImportSet(t *testing.T) {
 	}
 }
 
+func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
+	tests := []struct {
+		setRowIDs   []uint64
+		setColIDs   []uint64
+		setExp      map[uint64][]uint64
+		clearRowIDs []uint64
+		clearColIDs []uint64
+		clearExp    map[uint64][]uint64
+	}{
+		{
+			[]uint64{1, 1, 1, 1},
+			[]uint64{0, 1, 2, 3},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+			},
+			[]uint64{},
+			[]uint64{},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+			},
+		},
+		{
+			[]uint64{1, 1, 1, 1, 2, 2, 2, 2},
+			[]uint64{0, 1, 2, 3, 0, 1, 2, 3},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+				2: {0, 1, 2, 3},
+			},
+			[]uint64{1, 1, 2},
+			[]uint64{1, 2, 3},
+			map[uint64][]uint64{
+				1: {0, 3},
+				2: {0, 1, 2},
+			},
+		},
+		{
+			[]uint64{1, 1, 1, 1, 2},
+			[]uint64{0, 1, 2, 3, 1},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+				2: {1},
+			},
+			[]uint64{1, 1, 1, 1},
+			[]uint64{0, 1, 2, 3},
+			map[uint64][]uint64{
+				1: {},
+				2: {1},
+			},
+		},
+		{
+			[]uint64{1, 1, 1, 1, 2, 2, 1},
+			[]uint64{0, 1, 2, 3, 1, 8, 1},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+				2: {1, 8},
+			},
+			[]uint64{1, 1},
+			[]uint64{0, 0},
+			map[uint64][]uint64{
+				1: {1, 2, 3},
+				2: {1, 8},
+			},
+		},
+		{
+			[]uint64{1, 2, 3},
+			[]uint64{8, 8, 8},
+			map[uint64][]uint64{
+				1: {8},
+				2: {8},
+				3: {8},
+			},
+			[]uint64{1, 2, 3},
+			[]uint64{9, 9, 9},
+			map[uint64][]uint64{
+				1: {8},
+				2: {8},
+				3: {8},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) {
+			f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
+			_ = idx
+			defer f.Clean(t)
+
+			// Obtain transaction.
+			tx := f.txTestingOnly
+			defer tx.Rollback()
+
+			// Set import.
+			err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{})
+			if err != nil {
+				t.Fatalf("bulk importing ids: %v", err)
+			}
+
+			panicOn(tx.Commit())
+			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+
+			// Check for expected results.
+			for k, v := range test.setExp {
+				cols := f.mustRow(tx, k).Columns()
+				if !reflect.DeepEqual(cols, v) {
+					t.Fatalf("expected: %v, but got: %v", v, cols)
+				}
+			}
+
+			panicOn(tx.Commit())
+			tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+
+			// Clear import.
+			err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true})
+			if err != nil {
+				t.Fatalf("bulk clearing ids: %v", err)
+			}
+
+			panicOn(tx.Commit())
+			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+
+			// Check for expected results.
+			for k, v := range test.clearExp {
+				cols := f.mustRow(tx, k).Columns()
+				if !reflect.DeepEqual(cols, v) {
+					t.Fatalf("expected: %v, but got: %v", v, cols)
+				}
+			}
+		})
+	}
+}
+
 func TestFragment_ConcurrentImport(t *testing.T) {
 	t.Run("bulkImportStandard", func(t *testing.T) {
 		f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
@@ -2189,7 +2360,7 @@ func TestFragment_ConcurrentImport(t *testing.T) {
 
 		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		eg := errgroup.Group{}
 		eg.Go(func() error { return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) })
@@ -2291,7 +2462,7 @@ func TestFragment_ImportMutex(t *testing.T) {
 
 			// Obtain transaction.
 			tx := f.txTestingOnly
-			defer tx.Rollback() //LOOKATME
+			defer tx.Rollback()
 
 			// Set import.
 			err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{})
@@ -2324,6 +2495,141 @@ func TestFragment_ImportMutex(t *testing.T) {
 	}
 }
 
+// Ensure a fragment can import mutually exclusive values.
+// Now with Commits in the middle.
+func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
+	tests := []struct {
+		setRowIDs   []uint64
+		setColIDs   []uint64
+		setExp      map[uint64][]uint64
+		clearRowIDs []uint64
+		clearColIDs []uint64
+		clearExp    map[uint64][]uint64
+	}{
+		{
+			[]uint64{1, 1, 1, 1},
+			[]uint64{0, 1, 2, 3},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+			},
+			[]uint64{},
+			[]uint64{},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+			},
+		},
+		{
+			[]uint64{1, 1, 1, 1, 2, 2, 2, 2},
+			[]uint64{0, 1, 2, 3, 0, 1, 2, 3},
+			map[uint64][]uint64{
+				1: {},
+				2: {0, 1, 2, 3},
+			},
+			[]uint64{1, 1, 2},
+			[]uint64{1, 2, 3},
+			map[uint64][]uint64{
+				1: {},
+				2: {0, 1, 2},
+			},
+		},
+		{
+			[]uint64{1, 1, 1, 1, 2},
+			[]uint64{0, 1, 2, 3, 1},
+			map[uint64][]uint64{
+				1: {0, 2, 3},
+				2: {1},
+			},
+			[]uint64{1, 1, 1, 1},
+			[]uint64{0, 1, 2, 3},
+			map[uint64][]uint64{
+				1: {},
+				2: {1},
+			},
+		},
+		{
+			[]uint64{1, 1, 1, 1, 2, 2, 1},
+			[]uint64{0, 1, 2, 3, 1, 8, 1},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+				2: {8},
+			},
+			[]uint64{1, 1},
+			[]uint64{0, 0},
+			map[uint64][]uint64{
+				1: {1, 2, 3},
+				2: {8},
+			},
+		},
+		{
+			[]uint64{1, 2, 3},
+			[]uint64{8, 8, 8},
+			map[uint64][]uint64{
+				1: {},
+				2: {},
+				3: {8},
+			},
+			[]uint64{1, 2, 3},
+			[]uint64{9, 9, 9},
+			map[uint64][]uint64{
+				1: {},
+				2: {},
+				3: {8},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
+			f, idx := mustOpenMutexFragment("i", "f", viewStandard, 0, "")
+			defer f.Clean(t)
+
+			// Obtain transaction.
+			tx := f.txTestingOnly
+			defer tx.Rollback()
+
+			// Set import.
+			err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{})
+			if err != nil {
+				t.Fatalf("bulk importing ids: %v", err)
+			}
+
+			panicOn(tx.Commit())
+			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+
+			// Check for expected results.
+			for k, v := range test.setExp {
+				cols := f.mustRow(tx, k).Columns()
+				if !reflect.DeepEqual(cols, v) {
+					t.Fatalf("row: %d, expected: %v, but got: %v", k, v, cols)
+				}
+			}
+
+			panicOn(tx.Commit())
+			tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+
+			// Clear import.
+			err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true})
+			if err != nil {
+				t.Fatalf("bulk clearing ids: %v", err)
+			}
+
+			panicOn(tx.Commit())
+			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+
+			// Check for expected results.
+			for k, v := range test.clearExp {
+				cols := f.mustRow(tx, k).Columns()
+				if !reflect.DeepEqual(cols, v) {
+					t.Fatalf("row: %d expected: %v, but got: %v", k, v, cols)
+				}
+			}
+		})
+	}
+}
+
 // Ensure a fragment can import bool values.
 func TestFragment_ImportBool(t *testing.T) {
 	tests := []struct {
@@ -2415,7 +2721,7 @@ func TestFragment_ImportBool(t *testing.T) {
 
 			// Obtain transaction.
 			tx := f.txTestingOnly
-			defer tx.Rollback() //LOOKATME
+			defer tx.Rollback()
 
 			// Set import.
 			err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{})
@@ -2448,6 +2754,142 @@ func TestFragment_ImportBool(t *testing.T) {
 	}
 }
 
+// Ensure a fragment can import bool values, with Commits in between writes and reads.
+func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
+	tests := []struct {
+		setRowIDs   []uint64
+		setColIDs   []uint64
+		setExp      map[uint64][]uint64
+		clearRowIDs []uint64
+		clearColIDs []uint64
+		clearExp    map[uint64][]uint64
+	}{
+		{
+			[]uint64{1, 1, 1, 1},
+			[]uint64{0, 1, 2, 3},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+			},
+			[]uint64{},
+			[]uint64{},
+			map[uint64][]uint64{
+				1: {0, 1, 2, 3},
+			},
+		},
+		{
+			[]uint64{0, 0, 0, 0, 1, 1, 1, 1},
+			[]uint64{0, 1, 2, 3, 0, 1, 2, 3},
+			map[uint64][]uint64{
+				0: {},
+				1: {0, 1, 2, 3},
+			},
+			[]uint64{1, 1, 2},
+			[]uint64{1, 2, 3},
+			map[uint64][]uint64{
+				0: {},
+				1: {0, 3},
+				2: {},
+			},
+		},
+		{
+			[]uint64{0, 0, 0, 0, 1},
+			[]uint64{0, 1, 2, 3, 1},
+			map[uint64][]uint64{
+				0: {0, 2, 3},
+				1: {1},
+			},
+			[]uint64{1, 1, 1, 1},
+			[]uint64{0, 1, 2, 3},
+			map[uint64][]uint64{
+				0: {0, 2, 3},
+				1: {},
+			},
+		},
+		{
+			[]uint64{1, 1, 1, 1, 0, 0, 1},
+			[]uint64{0, 1, 2, 3, 1, 8, 1},
+			map[uint64][]uint64{
+				0: {8},
+				1: {0, 1, 2, 3},
+			},
+			[]uint64{1, 1},
+			[]uint64{0, 0},
+			map[uint64][]uint64{
+				0: {8},
+				1: {1, 2, 3},
+			},
+		},
+		{
+			[]uint64{0, 1, 2},
+			[]uint64{8, 8, 8},
+			map[uint64][]uint64{
+				0: {},
+				1: {}, // This isn't {8} because fragment doesn't validate bool values.
+				2: {8},
+			},
+			[]uint64{1, 2, 3},
+			[]uint64{9, 9, 9},
+			map[uint64][]uint64{
+				0: {},
+				1: {},
+				2: {8},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) {
+			f, idx := mustOpenBoolFragment("i", "f", viewStandard, 0, "")
+			_ = idx
+			defer f.Clean(t)
+
+			// Obtain transaction.
+			tx := f.txTestingOnly
+			defer tx.Rollback()
+
+			// Set import.
+			err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{})
+			if err != nil {
+				t.Fatalf("bulk importing ids: %v", err)
+			}
+
+			panicOn(tx.Commit())
+			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+
+			// Check for expected results.
+			for k, v := range test.setExp {
+				cols := f.mustRow(tx, k).Columns()
+				if !reflect.DeepEqual(cols, v) {
+					t.Fatalf("expected: %v, but got: %v", v, cols)
+				}
+			}
+
+			panicOn(tx.Commit())
+			tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+
+			// Clear import.
+			err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true})
+			if err != nil {
+				t.Fatalf("bulk importing ids: %v", err)
+			}
+
+			panicOn(tx.Commit())
+			tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+			defer tx.Rollback()
+
+			// Check for expected results.
+			for k, v := range test.clearExp {
+				cols := f.mustRow(tx, k).Columns()
+				if !reflect.DeepEqual(cols, v) {
+					t.Fatalf("expected: %v, but got: %v", v, cols)
+				}
+			}
+		})
+	}
+}
+
 func BenchmarkFragment_Snapshot(b *testing.B) {
 	if *FragmentPath == "" {
 		b.Skip("no fragment specified")
@@ -2478,10 +2920,6 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
 	_ = idx
 	defer f.Clean(b)
 
-	// Obtain transaction.
-	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
-
 	// Generate some intersecting data.
 	maxX := ShardWidth / 2
 	sz := maxX
@@ -2499,9 +2937,14 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
 			val += 2
 			i++
 		}
+
+		tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+		defer tx.Rollback()
+
 		if err := f.bulkImport(tx, rows, cols, options); err != nil {
 			b.Fatalf("Error Building Sample: %s", err)
 		}
+		panicOn(tx.Commit())
 		if row > max {
 			max = row
 		}
@@ -2544,14 +2987,16 @@ func BenchmarkFragment_Import(b *testing.B) {
 		copy(rowsUse, rows)
 		copy(colsUse, cols)
 		f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
-		_ = idx // Obtain transaction.
+		_ = idx
+		// Obtain transaction.
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 		b.StartTimer()
 		if err := f.bulkImport(tx, rowsUse, colsUse, options); err != nil {
 			b.Errorf("Error Building Sample: %s", err)
 		}
 		b.StopTimer()
+		tx.Rollback()
 		f.Clean(b)
 	}
 }
@@ -2645,6 +3090,10 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
 	}
 }
 func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
+	skipForBadger := os.Getenv("PILOSA_TXSRC") == "badger"
+	if skipForBadger {
+		b.Skip("skip for badger")
+	}
 	if testing.Short() {
 		b.SkipNow()
 	}
@@ -2664,6 +3113,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
 								// the cost of actually doing the op log for the large initial data set
 								// is excessive. force storage into snapshotted state, then use import
 								// to generate an op log and/or snapshot.
+								// note: skipped for badger, above.
 								_, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0)
 								if err != nil {
 									b.Fatalf("importing roaring: %v", err)
@@ -2719,7 +3169,7 @@ func BenchmarkImportStandard(b *testing.B) {
 					_ = idx
 					// Obtain transaction.
 					tx := f.txTestingOnly
-					defer tx.Rollback() //LOOKATME
+					defer tx.Rollback()
 
 					b.StartTimer()
 					err := f.bulkImport(tx, rowIDs, columnIDs, &ImportOptions{})
@@ -2727,6 +3177,7 @@ func BenchmarkImportStandard(b *testing.B) {
 						b.Errorf("import error: %v", err)
 					}
 					b.StopTimer()
+					tx.Rollback()
 					f.Clean(b)
 				}
 			})
@@ -2757,7 +3208,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
 						// to generate an op log and/or snapshot.
 						itr, err := roaring.NewRoaringIterator(data)
 						panicOn(err)
-						_, _, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, itr, false, false, 0)
+						_, _, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, itr, false, false, 0, nil)
 						if err != nil {
 							b.Errorf("import error: %v", err)
 						}
@@ -2877,14 +3328,21 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
 		}
 		origF.Close()
 		fi.Close()
-		nf := newFragment(NewHolder(DefaultPartitionN), fi.Name(), "i", "f", viewStandard, 0, 0)
+
+		h := NewHolder(DefaultPartitionN)
+		h.Path = fi.Name()
+		idx, err := h.CreateIndex("i", IndexOptions{})
+		panicOn(err)
+
+		nf := newFragment(h, fi.Name(), "i", "f", viewStandard, 0, 0)
 		err = nf.Open()
 		if err != nil {
 			b.Fatalf("opening fragment: %v", err)
 		}
 
 		// Obtain transaction.
-		tx := &RoaringTx{fragment: nf}
+		tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: nf})
+		defer tx.Rollback()
 
 		copy(rows, rowsOrig)
 		copy(cols, colsOrig)
@@ -2894,7 +3352,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
 		if err != nil {
 			b.Fatalf("bulkImport: %v", err)
 		}
-
+		panicOn(tx.Commit())
 		nf.Clean(b)
 	}
 }
@@ -2924,12 +3382,14 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) {
 
 		th := newTestHolder()
 		idx := fragTestMustOpenIndex("i", th, IndexOptions{})
+		if th.NeedsSnapshot() {
+			th.SnapshotQueue = newSnapshotQueue(1, 1, nil)
+		}
 		nf := newFragment(th, fi.Name(), "i", "f", viewStandard, 0, 0)
 
 		tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: nf})
 		defer tx.Rollback()
 
-		//nf := newFragment(NewHolder(DefaultPartitionN), fi.Name(), "i", "f", viewStandard, 0, 0)
 		err = nf.Open()
 		if err != nil {
 			b.Fatalf("opening fragment: %v", err)
@@ -3094,6 +3554,7 @@ func BenchmarkFileWrite(b *testing.B) {
 
 /////////////////////////////////////////////////////////////////////
 
+// not called under badger b/c f.idx.NeedsSnapshot() in Clean() avoids it.
 func (f *fragment) sanityCheck(t testing.TB) {
 	newBM := roaring.NewFileBitmap()
 	file, err := os.Open(f.path)
@@ -3110,6 +3571,7 @@ func (f *fragment) sanityCheck(t testing.TB) {
 		t.Fatalf("sanityCheck couldn't unmarshal fragment %s: %v", f.path, err)
 	}
 	// Refactor fragment.storage
+	// note: not called for badger, see above.
 	if equal, reason := newBM.BitwiseEqual(f.storage); !equal {
 		t.Fatalf("fragment %s: unmarshalled bitmap different: %v", f.path, reason)
 	}
@@ -3122,25 +3584,35 @@ func (f *fragment) Clean(t testing.TB) {
 	// check or else, in some cases, the background snapshot queue
 	// can decide to pick it up.
 	func() {
+		// should we skip snapshot queue stuff under badger/rbf?
 		defer f.mu.Unlock()
-		err := defaultSnapshotQueue.Await(f)
-		if err != nil {
-			t.Fatalf("snapshot failed before sanity check: %v", err)
-		}
-		f.sanityCheck(t)
-		if f.storage != nil && f.storage.Source != nil {
-			if f.storage.Source.Dead() {
-				t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID())
+
+		// badger doesn't need snapshot, so this stuff is skipped.
+		// The snapshot queue stuff doesn't work under badger.
+		if f.idx.NeedsSnapshot() {
+			err := defaultSnapshotQueue.Await(f)
+			if err != nil {
+				t.Fatalf("snapshot failed before sanity check: %v", err)
+			}
+			f.sanityCheck(t)
+			if f.storage != nil && f.storage.Source != nil {
+				if f.storage.Source.Dead() {
+					t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID())
+				}
 			}
 		}
 	}()
 	if f.txTestingOnly != nil {
 		f.txTestingOnly.Rollback()
+		panicOn(f.idx.Txf.CloseIndex(f.idx))
 	}
 	errc := f.Close()
 	// prevent double-closes of generation during testing.
 	f.gen = nil
-	errf := os.Remove(f.path)
+	var errf error
+	if FileExists(f.path) {
+		errf = os.Remove(f.path) // remove /var/folders/2x/hm9gp5ys3k9gmm5f_vzm_6wc0000gn/T/pilosa-index-768377904/i/f/views/standard/fragments/0: no such file or directory
+	}
 	errp := os.Remove(f.cachePath())
 	if errc != nil || errf != nil {
 		t.Fatal("cleaning up fragment: ", errc, errf, errp)
@@ -3180,15 +3652,9 @@ func mustOpenBSIFragment(index, field, view string, shard uint64) (*fragment, *I
 	return mustOpenFragmentFlags(index, field, view, shard, "", 1)
 }
 
-var testHolder = NewHolder(DefaultPartitionN)
-
-func init() {
-	testHolder.SnapshotQueue = newSnapshotQueue(1, 1, nil)
-}
-
 func newTestHolder() *Holder {
 	h := NewHolder(DefaultPartitionN)
-	h.SnapshotQueue = newSnapshotQueue(1, 1, nil)
+	//h.SnapshotQueue = newSnapshotQueue(1, 1, nil)
 	return h
 }
 
@@ -3199,13 +3665,15 @@ func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Inde
 		panic(err)
 	}
 	holder.Path = path
+	holder.mu.Lock()
 	idx, err := holder.createIndex(index, opt)
+	holder.mu.Unlock()
 	panicOn(err)
 
 	idx.keys = opt.Keys
 	idx.trackExistence = opt.TrackExistence
 
-	if err := idx.Open(); err != nil {
+	if err := idx.Open(false); err != nil {
 		panic(err)
 	}
 	return idx
@@ -3226,13 +3694,14 @@ func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType st
 	// new:
 	th := newTestHolder()
 	idx := fragTestMustOpenIndex(index, th, IndexOptions{})
+	if th.NeedsSnapshot() {
+		th.SnapshotQueue = newSnapshotQueue(1, 1, nil)
+	}
 	f := newFragment(th, file.Name(), index, field, view, shard, flags)
 
 	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
 	f.txTestingOnly = tx
 
-	//old: f := newFragment(testHolder, file.Name(), index, field, view, shard, flags)
-
 	f.CacheType = cacheType
 	f.RowAttrStore = &memAttrStore{
 		store: make(map[uint64]map[string]interface{}),
@@ -3292,9 +3761,9 @@ func TestFragment_RowsIteration(t *testing.T) {
 	t.Run("firstContainer", func(t *testing.T) {
 		f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
 		_ = idx
-		defer f.Clean(t)
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
+		defer f.Clean(t)
 
 		expectedAll := make([]uint64, 0)
 		expectedOdd := make([]uint64, 0)
@@ -3326,9 +3795,9 @@ func TestFragment_RowsIteration(t *testing.T) {
 	t.Run("secondRow", func(t *testing.T) {
 		f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
 		_ = idx
-		defer f.Clean(t)
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
+		defer f.Clean(t)
 
 		expected := []uint64{1, 2}
 		if _, err := f.setBit(tx, 1, 66000); err != nil {
@@ -3357,9 +3826,9 @@ func TestFragment_RowsIteration(t *testing.T) {
 	t.Run("combinations", func(t *testing.T) {
 		f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
 		_ = idx
-		defer f.Clean(t)
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
+		defer f.Clean(t)
 
 		expectedRows := make([]uint64, 0)
 		for r := uint64(1); r < uint64(10000); r += 250 {
@@ -3602,9 +4071,9 @@ func TestFragmentRowIterator(t *testing.T) {
 	t.Run("basic", func(t *testing.T) {
 		f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
 		_ = idx
-		defer f.Clean(t)
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
+		defer f.Clean(t)
 
 		f.mustSetBits(tx, 0, 0)
 		f.mustSetBits(tx, 1, 0)
@@ -3650,7 +4119,7 @@ func TestFragmentRowIterator(t *testing.T) {
 		_ = idx
 		defer f.Clean(t)
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		f.mustSetBits(tx, 1, 0)
 		f.mustSetBits(tx, 3, 0)
@@ -3696,7 +4165,7 @@ func TestFragmentRowIterator(t *testing.T) {
 		_ = idx
 		defer f.Clean(t)
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		f.mustSetBits(tx, 0, 0)
 		f.mustSetBits(tx, 1, 0)
@@ -3731,7 +4200,7 @@ func TestFragmentRowIterator(t *testing.T) {
 		_ = idx
 		defer f.Clean(t)
 		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
+		defer tx.Rollback()
 
 		f.mustSetBits(tx, 1, 0)
 		f.mustSetBits(tx, 3, 0)
@@ -3762,7 +4231,194 @@ func TestFragmentRowIterator(t *testing.T) {
 	})
 }
 
+// same, with commits
+func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
+	t.Run("basic", func(t *testing.T) {
+		f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
+		_ = idx
+		tx := f.txTestingOnly
+		defer tx.Rollback()
+		defer f.Clean(t)
+
+		f.mustSetBits(tx, 0, 0)
+		f.mustSetBits(tx, 1, 0)
+		f.mustSetBits(tx, 2, 0)
+		f.mustSetBits(tx, 3, 0)
+
+		panicOn(tx.Commit())
+		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+		defer tx.Rollback()
+
+		iter, err := f.rowIterator(tx, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := uint64(0); i < 4; i++ {
+			row, id, _, wrapped, err := iter.Next()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if id != i {
+				t.Fatalf("expected row %d but got %d", i, id)
+			}
+			if wrapped {
+				t.Fatalf("shouldn't have wrapped")
+			}
+			if !reflect.DeepEqual(row.Columns(), []uint64{0}) {
+				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
+			}
+		}
+		row, id, _, wrapped, err := iter.Next()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if row != nil {
+			t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns())
+		}
+		if id != 0 {
+			t.Fatalf("id should be 0 after iterator is exhausted, got %d", id)
+		}
+		if !wrapped {
+			t.Fatalf("wrapped should be true after iterator is exhausted")
+		}
+	})
+
+	t.Run("skipped rows", func(t *testing.T) {
+		f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
+		_ = idx
+		defer f.Clean(t)
+		tx := f.txTestingOnly
+		defer tx.Rollback()
+
+		f.mustSetBits(tx, 1, 0)
+		f.mustSetBits(tx, 3, 0)
+		f.mustSetBits(tx, 5, 0)
+		f.mustSetBits(tx, 7, 0)
+
+		panicOn(tx.Commit())
+		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+		defer tx.Rollback()
+
+		iter, err := f.rowIterator(tx, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := uint64(1); i < 8; i += 2 {
+			row, id, _, wrapped, err := iter.Next()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if id != i {
+				t.Fatalf("expected row %d but got %d", i, id)
+			}
+			if wrapped {
+				t.Fatalf("shouldn't have wrapped")
+			}
+			if !reflect.DeepEqual(row.Columns(), []uint64{0}) {
+				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
+			}
+		}
+		row, id, _, wrapped, err := iter.Next()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if row != nil {
+			t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns())
+		}
+		if id != 0 {
+			t.Fatalf("id should be 0 after iterator is exhausted, got %d", id)
+		}
+		if !wrapped {
+			t.Fatalf("wrapped should be true after iterator is exhausted")
+		}
+	})
+
+	t.Run("basic wrapped", func(t *testing.T) {
+		f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
+		_ = idx
+		defer f.Clean(t)
+		tx := f.txTestingOnly
+		defer tx.Rollback()
+
+		f.mustSetBits(tx, 0, 0)
+		f.mustSetBits(tx, 1, 0)
+		f.mustSetBits(tx, 2, 0)
+		f.mustSetBits(tx, 3, 0)
+
+		panicOn(tx.Commit())
+		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+		defer tx.Rollback()
+
+		iter, err := f.rowIterator(tx, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := uint64(0); i < 5; i++ {
+			row, id, _, wrapped, err := iter.Next()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if id != i%4 {
+				t.Fatalf("expected row %d but got %d", i%4, id)
+			}
+			if wrapped && i < 4 {
+				t.Fatalf("shouldn't have wrapped")
+			} else if !wrapped && i >= 4 {
+				t.Fatalf("should have wrapped")
+			}
+			if !reflect.DeepEqual(row.Columns(), []uint64{0}) {
+				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
+			}
+		}
+	})
+
+	t.Run("skipped rows wrapped", func(t *testing.T) {
+		f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
+		_ = idx
+		defer f.Clean(t)
+		tx := f.txTestingOnly
+		defer tx.Rollback()
+
+		f.mustSetBits(tx, 1, 0)
+		f.mustSetBits(tx, 3, 0)
+		f.mustSetBits(tx, 5, 0)
+		f.mustSetBits(tx, 7, 0)
+
+		panicOn(tx.Commit())
+		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+		defer tx.Rollback()
+
+		iter, err := f.rowIterator(tx, true)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := uint64(1); i < 10; i += 2 {
+			row, id, _, wrapped, err := iter.Next()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if id != i%8 {
+				t.Errorf("expected row %d but got %d", i%8, id)
+			}
+			if wrapped && i < 8 {
+				t.Errorf("shouldn't have wrapped")
+			} else if !wrapped && i >= 8 {
+				t.Errorf("should have wrapped")
+			}
+			if !reflect.DeepEqual(row.Columns(), []uint64{0}) {
+				t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns())
+			}
+		}
+	})
+}
+
 func TestUnionInPlaceMapped(t *testing.T) {
+
+	skipForBadger := os.Getenv("PILOSA_TXSRC") == "badger"
+	if skipForBadger {
+		t.Skip("skip for badger")
+	}
+
 	f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
 	// note: clean has to be deferred first, because it has to run with
 	// the lock *not* held, because it is sometimes so it has to grab the
@@ -3928,7 +4584,7 @@ func TestIntLTRegression(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	_, err := f.setValue(tx, 1, 6, 33)
 	if err != nil {
@@ -3961,7 +4617,8 @@ func TestFragmentBSIUnsigned(t *testing.T) {
 	defer f.Clean(t)
 
 	// Obtain transaction.
-	tx := &RoaringTx{fragment: f}
+	tx := f.txTestingOnly
+	defer tx.Rollback()
 
 	// Number of bits to test.
 	const k = 6
@@ -4116,13 +4773,181 @@ func TestFragmentBSIUnsigned(t *testing.T) {
 	})
 }
 
+// same, WithTxCommit version
+func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) {
+	f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
+	_ = idx
+	defer f.Clean(t)
+
+	// Obtain transaction.
+	tx := f.txTestingOnly
+	defer tx.Rollback()
+
+	// Number of bits to test.
+	const k = 6
+
+	// Load all numbers into an effectively diagonal matrix.
+	for i := 0; i < 1<<k; i++ {
+		ok, err := f.setValue(tx, uint64(i), k, int64(i))
+		if err != nil {
+			t.Fatalf("failed to set col %d to %d: %v", uint64(i), int64(i), err)
+		} else if !ok {
+			t.Fatalf("no change when setting col %d to %d", uint64(i), int64(i))
+		}
+	}
+
+	panicOn(tx.Commit())
+	tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+	defer tx.Rollback()
+
+	// Generate a list of columns.
+	cols := make([]uint64, 1<<k)
+	for i := range cols {
+		cols[i] = uint64(i)
+	}
+
+	// Minimum and maximum checking bounds.
+	// This is mostly arbitrary.
+	minCheck, maxCheck := -3, 1<<(k+1)
+
+	t.Run("<", func(t *testing.T) {
+		for i := minCheck; i < maxCheck; i++ {
+			row, err := f.rangeLT(tx, k, int64(i), false)
+			if err != nil {
+				t.Fatalf("failed to query fragment: %v", err)
+			}
+			var expect []uint64
+			switch {
+			case i < 0:
+			case i < len(cols):
+				expect = cols[:i]
+			default:
+				expect = cols
+			}
+			got := row.Columns()
+			if !sliceEq(expect, got) {
+				t.Errorf("expected %v but got %v for x < %d", expect, got, i)
+			}
+		}
+	})
+	t.Run("<=", func(t *testing.T) {
+		for i := minCheck; i < maxCheck; i++ {
+			row, err := f.rangeLT(tx, k, int64(i), true)
+			if err != nil {
+				t.Fatalf("failed to query fragment: %v", err)
+			}
+			var expect []uint64
+			switch {
+			case i < 0:
+			case i < len(cols)-1:
+				expect = cols[:i+1]
+			default:
+				expect = cols
+			}
+			got := row.Columns()
+			if !sliceEq(expect, got) {
+				t.Errorf("expected %v but got %v for x <= %d", expect, got, i)
+			}
+		}
+	})
+	t.Run(">", func(t *testing.T) {
+		for i := minCheck; i < maxCheck; i++ {
+			row, err := f.rangeGT(tx, k, int64(i), false)
+			if err != nil {
+				t.Fatalf("failed to query fragment: %v", err)
+			}
+			var expect []uint64
+			switch {
+			case i < 0:
+				expect = cols
+			case i < len(cols)-1:
+				expect = cols[i+1:]
+			default:
+			}
+			got := row.Columns()
+			if !sliceEq(expect, got) {
+				t.Errorf("expected %v but got %v for x > %d", expect, got, i)
+			}
+		}
+	})
+	t.Run(">=", func(t *testing.T) {
+		for i := minCheck; i < maxCheck; i++ {
+			row, err := f.rangeGT(tx, k, int64(i), true)
+			if err != nil {
+				t.Fatalf("failed to query fragment: %v", err)
+			}
+			var expect []uint64
+			switch {
+			case i < 0:
+				expect = cols
+			case i < len(cols):
+				expect = cols[i:]
+			default:
+			}
+			got := row.Columns()
+			if !sliceEq(expect, got) {
+				t.Errorf("expected %v but got %v for x >= %d", expect, got, i)
+			}
+		}
+	})
+	t.Run("Range", func(t *testing.T) {
+		for i := minCheck; i < maxCheck; i++ {
+			for j := i; j < maxCheck; j++ {
+				row, err := f.rangeBetween(tx, k, int64(i), int64(j))
+				if err != nil {
+					t.Fatalf("failed to query fragment: %v", err)
+				}
+				var lower, upper int
+				switch {
+				case i < 0:
+					lower = 0
+				case i > len(cols):
+					lower = len(cols)
+				default:
+					lower = i
+				}
+				switch {
+				case j < 0:
+					upper = 0
+				case j >= len(cols):
+					upper = len(cols)
+				default:
+					upper = j + 1
+				}
+				expect := cols[lower:upper]
+				got := row.Columns()
+				if !sliceEq(expect, got) {
+					t.Errorf("expected %v but got %v for %d <= x <= %d", expect, got, i, j)
+				}
+			}
+		}
+	})
+	t.Run("==", func(t *testing.T) {
+		for i := minCheck; i < maxCheck; i++ {
+			row, err := f.rangeEQ(tx, k, int64(i))
+			if err != nil {
+				t.Fatalf("failed to query fragment: %v", err)
+			}
+			var expect []uint64
+			if i >= 0 && i < len(cols) {
+				expect = cols[i : i+1]
+			}
+			got := row.Columns()
+			if !sliceEq(expect, got) {
+				t.Errorf("expected %v but got %v for x == %d", expect, got, i)
+			}
+		}
+	})
+}
+
 func TestFragmentBSISigned(t *testing.T) {
 	f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
 	_ = idx
 	defer f.Clean(t)
 
 	// Obtain transaction.
-	tx := &RoaringTx{fragment: f}
+	tx := f.txTestingOnly
+	defer tx.Rollback()
 
 	// Number of bits to test.
 	const k = 6
@@ -4365,7 +5190,18 @@ func TestImportClearRestart(t *testing.T) {
 
 				check(t, tx, f, exp)
 
-				f2 := newFragment(NewHolder(DefaultPartitionN), f.path, "i", "f", viewStandard, 0, 0)
+				h := NewHolder(DefaultPartitionN)
+				h.Path = filepath.Dir(f.path)
+				idx2, err := h.CreateIndex("i", IndexOptions{})
+				_ = idx2
+				panicOn(err)
+
+				// OVERWRITING the f.path with a new fragment
+				f2 := newFragment(h, f.path, "i", "f", viewStandard, 0, 0)
+
+				//				f2, idx2 := mustOpenFragment("i", "f", viewStandard, 0, "")
+				//				_ = idx2
+
 				f2.MaxOpN = maxOpN
 				f2.CacheType = f.CacheType
 
@@ -4408,7 +5244,13 @@ func TestImportClearRestart(t *testing.T) {
 
 				panicOn(tx2.Commit())
 
-				f3 := newFragment(NewHolder(DefaultPartitionN), f2.path, "i", "f", viewStandard, 0, 0)
+				h3 := NewHolder(DefaultPartitionN)
+				h3.Path = filepath.Dir(f2.path)
+				idx3, err := h3.CreateIndex("i", IndexOptions{})
+				_ = idx3
+				panicOn(err)
+
+				f3 := newFragment(h3, f2.path, "i", "f", viewStandard, 0, 0)
 				f3.MaxOpN = maxOpN
 				f3.CacheType = f.CacheType
 
@@ -4529,13 +5371,18 @@ func TestImportMultipleValues(t *testing.T) {
 
 				// Obtain transaction.
 				tx := f.txTestingOnly
-				defer tx.Rollback() //LOOKATME
+				defer tx.Rollback()
 
 				err := f.importValue(tx, test.cols, test.vals, test.depth, false)
 				if err != nil {
 					t.Fatalf("importing values: %v", err)
 				}
 
+				// probably too slow, would hit disk alot:
+				//panicOn(tx.Commit())
+				//tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+				//defer tx.Rollback()
+
 				for i := range test.checkCols {
 					cc, cv := test.checkCols[i], test.checkVals[i]
 					n, exists, err := f.value(tx, cc, test.depth)
@@ -4592,7 +5439,7 @@ func TestImportValueRowCache(t *testing.T) {
 
 				// Obtain transaction.
 				tx := f.txTestingOnly
-				defer tx.Rollback() //LOOKATME
+				defer tx.Rollback()
 
 				// First import (tc1)
 				if err := f.importValue(tx, test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil {
@@ -4625,12 +5472,13 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
 	_ = idx
 	defer f.Clean(t)
 
+	// Obtain transaction, but don't start another b/c the
+	// two goroutines below need the same view.
+	tx := f.txTestingOnly
+	defer tx.Rollback()
+
 	eg := &errgroup.Group{}
 	eg.Go(func() error {
-		// Obtain transaction.
-		tx := f.txTestingOnly
-		defer tx.Rollback() //LOOKATME
-
 		for i := uint64(0); i < 1000; i++ {
 			_, err := f.setBit(tx, i%4, i)
 			if err != nil {
@@ -4640,10 +5488,6 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
 		return nil
 	})
 
-	// Obtain transaction.
-	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
-
 	acc := uint64(0)
 	for i := uint64(0); i < 100; i++ {
 		r := f.mustRow(tx, i%4)
@@ -4659,6 +5503,7 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
 func TestRemapCache(t *testing.T) {
 	f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
 	_ = idx
+	index, field, view, shard := f.index, f.field, f.view, f.shard
 
 	// request a panic that doesn't kill the program on fault
 	wouldFault := debug.SetPanicOnFault(true)
@@ -4678,10 +5523,10 @@ func TestRemapCache(t *testing.T) {
 
 	// Obtain transaction.
 	tx := f.txTestingOnly
-	defer tx.Rollback() //LOOKATME
+	defer tx.Rollback()
 
 	// create a container
-	_, err := f.storage.Add(65537)
+	_, err := tx.Add(index, field, view, shard, !doBatched, 65537)
 	if err != nil {
 		t.Fatalf("storage add: %v", err)
 	}
@@ -4694,7 +5539,7 @@ func TestRemapCache(t *testing.T) {
 	_ = f.mustRow(tx, 0)
 	// add a bit that isn't in that container, so that container doesn't
 	// change
-	_, err = f.storage.Add(2)
+	_, err = tx.Add(index, field, view, shard, !doBatched, 2)
 	if err != nil {
 		t.Fatalf("storage add: %v", err)
 	}
diff --git a/generation.go b/generation.go
index e9c4e93c5..268585b1d 100644
--- a/generation.go
+++ b/generation.go
@@ -261,6 +261,7 @@ func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
 	if err != nil {
 		return false, err
 	}
+
 	// do we actually want this in every openFile? I don't know.
 	if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
 		_ = syswrap.CloseFile(m.file)
@@ -438,3 +439,25 @@ func newGeneration(existing generation, path string, readData bool, setup func([
 	// does get cleaned up.
 	return &m, nil
 }
+
+// NopGeneration is used in fragment.openStorage() to short-circuit
+// generation stuff that only applies to RoaringTx; doesn't apply to RBFTx/BadgerTx/etc.
+type NopGeneration struct {
+}
+
+func (g *NopGeneration) Transaction(w *io.Writer, f func() error) error {
+	return f()
+}
+func (g *NopGeneration) Done() {}
+func (g *NopGeneration) Generation() int64 {
+	return 0
+}
+func (g *NopGeneration) ID() string {
+	return "NOP"
+}
+func (g *NopGeneration) Dead() bool {
+	return true
+}
+func (g *NopGeneration) Bytes() (ret []byte) {
+	return
+}
diff --git a/generation_test.go b/generation_test.go
index 7df71fc98..bc0a1854f 100644
--- a/generation_test.go
+++ b/generation_test.go
@@ -53,8 +53,11 @@ func TestGenerationPanic(t *testing.T) {
 	if unsafe.Pointer(&prevData[0]) == unsafe.Pointer(&newData[0]) {
 		t.Fatalf("test can't run usefully, didn't get new data pointer")
 	}
-
-	err := f.gen.Transaction(&f.storage.OpWriter, func() error {
+	var wp *io.Writer
+	if f.storage != nil {
+		wp = &f.storage.OpWriter
+	}
+	err := f.gen.Transaction(wp, func() error {
 		prevData[0] = 0x3c
 		return nil
 	})
diff --git a/holder.go b/holder.go
index 58dca7f2c..e74672d31 100644
--- a/holder.go
+++ b/holder.go
@@ -528,9 +528,9 @@ func (h *Holder) Open() error {
 
 		if h.isCoordinator() {
 			index.createdAt = timestamp()
-			err = index.OpenWithTimestamp()
+			err = index.OpenWithTimestamp(false)
 		} else {
-			err = index.Open()
+			err = index.Open(false)
 		}
 		if err != nil {
 			if err == ErrName {
@@ -626,6 +626,17 @@ func (h *Holder) BeginTx(writable bool, index *Index) (Tx, error) {
 	return index.Txf.NewTx(Txo{Write: writable, Index: index}), nil
 }
 
+func (h *Holder) NeedsSnapshot() bool {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	for _, idx := range h.indexes {
+		if idx.NeedsSnapshot() {
+			return true
+		}
+	}
+	return false
+}
+
 // HasData returns true if Holder contains at least one index.
 // This is used to determine if the rebalancing of data is necessary
 // when a node joins the cluster.
@@ -802,7 +813,20 @@ func (h *Holder) applyCreatedAt(indexes []*IndexInfo) {
 }
 
 // IndexPath returns the path where a given index is stored.
-func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
+func (h *Holder) IndexPath(name string) string {
+	return filepath.Join(h.Path, name)
+}
+
+// HolderPathFromIndexPath is
+// used by test/index.go:71 in test.Index.Reopen() to get the right
+// path into a test Holder that doesn't know its own proper path.
+// If the Holder changes index paths to being something other than
+// holderPath + "/" + indexName, this will need adjusting too.
+func (h *Holder) HolderPathFromIndexPath(indexPath, indexName string) string {
+	n := len(indexPath)
+	hpath2 := indexPath[:n-(len(indexName)+1)]
+	return hpath2
+}
 
 // Index returns the index by name.
 func (h *Holder) Index(name string) *Index {
@@ -811,7 +835,9 @@ func (h *Holder) Index(name string) *Index {
 	return h.index(name)
 }
 
-func (h *Holder) index(name string) *Index { return h.indexes[name] }
+func (h *Holder) index(name string) *Index {
+	return h.indexes[name]
+}
 
 // Indexes returns a list of all indexes in the holder.
 func (h *Holder) Indexes() []*Index {
@@ -867,7 +893,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
 	index.keys = opt.Keys
 	index.trackExistence = opt.TrackExistence
 
-	if err = index.Open(); err != nil {
+	if err = index.Open(true); err != nil {
 		return nil, errors.Wrap(err, "opening")
 	}
 	if err = index.saveMeta(); err != nil {
@@ -1799,3 +1825,15 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) {
 	}
 	return nil
 }
+
+// used by Index.openFields(), enabling Tx / Txf by telling
+// the holder about its own indexes.
+func (h *Holder) addIndexFromField(idx *Index) {
+	h.mu.Lock()
+	h.indexes[idx.Name()] = idx
+	h.mu.Unlock()
+}
+
+func (h *Holder) unprotectedAddIndexFromField(idx *Index) {
+	h.indexes[idx.Name()] = idx
+}
diff --git a/holder_test.go b/holder_test.go
index a527307ba..4280dadbd 100644
--- a/holder_test.go
+++ b/holder_test.go
@@ -32,6 +32,8 @@ import (
 )
 
 func TestHolder_Open(t *testing.T) {
+	skipForBadger := os.Getenv("PILOSA_TXSRC") == "badger"
+
 	t.Run("ErrIndexName", func(t *testing.T) {
 		h := test.MustOpenHolder()
 
@@ -165,6 +167,9 @@ func TestHolder_Open(t *testing.T) {
 	})
 
 	t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
+		if skipForBadger {
+			t.Skip("skipping for badger")
+		}
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
@@ -201,6 +206,10 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
+		if skipForBadger {
+			t.Skip("skipping for badger")
+		}
+
 		h := test.MustOpenHolder()
 		defer h.Close()
 
@@ -233,6 +242,10 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
+		if skipForBadger {
+			t.Skip("skipping for badger")
+		}
+
 		h := test.MustOpenHolder()
 		defer h.Close()
 
@@ -723,24 +736,17 @@ func TestHolderSyncer_IntField(t *testing.T) {
 		hldr0.SetValue("i", "f", 1, 1)
 
 		// in c0 expect the 1 bit
-		//idx0.Dump("in c0, before SyncData")
 
 		// Set data on node1. columnID=2, value=2
 		idx1 := hldr1.SetValue("i", "f", 2, 2)
 		_ = idx1
 
-		//idx1.Dump("in c1, before SyncData")
-
-		//vv("before c[0] SyncData")
 		err = c[0].Server.SyncData()
 		if err != nil {
 			t.Fatalf("syncing node 0: %v", err)
 		}
-		//vv("after c[0] SyncData")
 
 		// expect 3 rows, the 1 bit + 2 rows for the 2 value as BSI. But, we only see that c0 overwrote c1.
-		//idx0.Dump("in c0, after syncData")
-		//idx1.Dump("in c1, after syncData")
 
 		// Problem is: data at c1 was replaced by c0, instead of being merged with existing c1.
 		// Problem is: data at c0 did not receive and merge the c1 data.
@@ -810,8 +816,6 @@ func TestHolderSyncer_IntField(t *testing.T) {
 		}
 
 		// dump the badger keys for both c0 and c1
-		//vv("in c0, allkeys = '%v'", idx0.StringifiedBadgerKeys(nil))
-		//vv("in c1, allkeys = '%v'", c[1].index.StringifiedBadgerKeys())
 
 		// Verify data is the same on both nodes.
 		for i, hldr := range []*test.Holder{hldr0, hldr1} {
diff --git a/index.go b/index.go
index 1474616bc..f9c6d2099 100644
--- a/index.go
+++ b/index.go
@@ -99,16 +99,16 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
 		}
 	}
 
-	txf, err := newTxFactory(txsrc, path)
-	if err != nil {
-		return nil, errors.Wrap(err, "creating newTxFactory")
-	}
-
-	err = validateName(name)
+	err := validateName(name)
 	if err != nil {
 		return nil, errors.Wrap(err, "validating name")
 	}
 
+	txf, err := NewTxFactory(txsrc, holder.Path, name)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating newTxFactory")
+	}
+
 	idx := &Index{
 		path:   path,
 		name:   name,
@@ -134,6 +134,14 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
 	return idx, nil
 }
 
+func (i *Index) NewTx(txo Txo) Tx {
+	return i.Txf.NewTx(txo)
+}
+
+func (i *Index) NeedsSnapshot() bool {
+	return i.Txf.NeedsSnapshot()
+}
+
 // CreatedAt is an timestamp for a specific version of an index.
 func (i *Index) CreatedAt() int64 {
 	i.mu.RLock()
@@ -148,7 +156,9 @@ func (i *Index) Name() string { return i.name }
 func (i *Index) QualifiedName() string { return i.qualifiedName }
 
 // Path returns the path the index was initialized with.
-func (i *Index) Path() string { return i.path }
+func (i *Index) Path() string {
+	return i.path
+}
 
 // TranslateStorePath returns the translation database path for a partition.
 func (i *Index) TranslateStorePath(partitionID int) string {
@@ -181,12 +191,12 @@ func (i *Index) options() IndexOptions {
 }
 
 // Open opens and initializes the index.
-func (i *Index) Open() error { return i.open(false) }
+func (i *Index) Open(haveHolderLock bool) error { return i.open(false, haveHolderLock) }
 
 // OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields.
-func (i *Index) OpenWithTimestamp() error { return i.open(true) }
+func (i *Index) OpenWithTimestamp(haveHolderLock bool) error { return i.open(true, haveHolderLock) }
 
-func (i *Index) open(withTimestamp bool) (err error) {
+func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) {
 	// Ensure the path exists.
 	i.holder.Logger.Debugf("ensure index path exists: %s", i.path)
 	if err := os.MkdirAll(i.path, 0777); err != nil {
@@ -200,7 +210,7 @@ func (i *Index) open(withTimestamp bool) (err error) {
 	}
 
 	i.holder.Logger.Debugf("open fields for index: %s", i.name)
-	if err := i.openFields(withTimestamp); err != nil {
+	if err := i.openFields(withTimestamp, haveHolderLock); err != nil {
 		return errors.Wrap(err, "opening fields")
 	}
 
@@ -243,7 +253,7 @@ func (i *Index) open(withTimestamp bool) (err error) {
 var indexQueue = make(chan struct{}, 8)
 
 // openFields opens and initializes the fields inside the index.
-func (i *Index) openFields(withTimestamp bool) error {
+func (i *Index) openFields(withTimestamp, haveHolderLock bool) error {
 	f, err := os.Open(i.path)
 	if err != nil {
 		return errors.Wrap(err, "opening directory")
@@ -273,7 +283,31 @@ fileLoop:
 					<-indexQueue
 				}()
 				i.holder.Logger.Debugf("open field: %s", fi.Name())
+
 				mu.Lock()
+
+				// i.holder needs to know about its index i for the Txf to work.
+				//
+				// We face either a deadlock or a race here.
+				//
+				// We get a deadlock in TestIndex_CreateField/"BSIFields"/"OK"
+				// if we call addIndexFromField, because in that test
+				// we get here while already holding i.holder.mu.
+				//
+				// On the other had, we get races on other tests
+				// such as TestExecutor_Execute_Existence/Row
+				// if we call unprotectedAddIndexFromField which does
+				// not lock i.holder.mu.
+				//
+				// The resolution was to have the goroutines that are holding
+				// the lock already tell us. That is the haveHolderLock
+				// argument.
+				if haveHolderLock {
+					i.holder.unprotectedAddIndexFromField(i)
+				} else {
+					i.holder.addIndexFromField(i)
+				}
+
 				fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
 				if withTimestamp {
 					fld.createdAt = timestamp()
@@ -287,6 +321,7 @@ fileLoop:
 				// up a foreign index.
 				fld.holder = i.holder
 
+				// open all the views
 				if err := fld.Open(); err != nil {
 					return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
 				}
@@ -546,6 +581,9 @@ func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) {
 	// Add to index's field lookup.
 	i.fields[name] = f
 
+	// enable Txf to find the index in field_test.go TestField_SetValue
+	f.idx = i
+
 	// Kick off the field's translation sync process.
 	if err := i.translationSyncer.Reset(); err != nil {
 		return nil, errors.Wrap(err, "resetting translation syncer")
@@ -559,6 +597,7 @@ func (i *Index) newField(path, name string) (*Field, error) {
 	if err != nil {
 		return nil, err
 	}
+	f.idx = i
 	f.Stats = i.Stats
 	f.broadcaster = i.broadcaster
 	f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
diff --git a/index_internal_test.go b/index_internal_test.go
index 5b5a5b5a3..4f569c745 100644
--- a/index_internal_test.go
+++ b/index_internal_test.go
@@ -25,7 +25,10 @@ func mustOpenIndex(opt IndexOptions) *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := NewIndex(NewHolder(1), path, "i")
+	h := NewHolder(1)
+	h.Path = path
+	index, err := h.CreateIndex("i", opt)
+
 	if err != nil {
 		panic(err)
 	}
@@ -33,7 +36,7 @@ func mustOpenIndex(opt IndexOptions) *Index {
 	index.keys = opt.Keys
 	index.trackExistence = opt.TrackExistence
 
-	if err := index.Open(); err != nil {
+	if err := index.Open(false); err != nil {
 		panic(err)
 	}
 	return index
@@ -44,7 +47,7 @@ func (i *Index) reopen() error {
 	if err := i.Close(); err != nil {
 		return err
 	}
-	if err := i.Open(); err != nil {
+	if err := i.Open(false); err != nil {
 		return err
 	}
 	return nil
diff --git a/mmap_test.go b/mmap_test.go
index 20fbd7685..43321cea7 100644
--- a/mmap_test.go
+++ b/mmap_test.go
@@ -32,11 +32,11 @@ type cv struct {
 func forceSnapshotsCheckMapping(t *testing.T) {
 	depth := uint(6)
 	f, idx := mustOpenBSIFragment("i", "f", viewStandard, 0)
-	_ = idx
 	f.Logger = logger.NewLogfLogger(t)
 	defer f.Clean(t)
 
-	tx := &RoaringTx{fragment: f}
+	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
+	defer tx.Rollback()
 
 	for i := 0; i < f.MaxOpN; i++ {
 		_, _ = f.setBit(tx, 0, uint64(32*i))
diff --git a/pql/ast.go b/pql/ast.go
index 8ebfaf122..b150cd3c5 100644
--- a/pql/ast.go
+++ b/pql/ast.go
@@ -22,8 +22,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/molecula/ext"
 )
 
 // Query represents a PQL query.
@@ -345,7 +343,8 @@ var callInfoByFunc = map[string]callInfo{
 	"Row":    {allowUnknown: true},
 	"Range":  {allowUnknown: true},
 
-	"Distinct": {allowUnknown: true},
+	"Distinct":  {allowUnknown: true, callType: PrecallGlobal},
+	"Condition": {allowUnknown: true},
 
 	// allow only "field=X" cases with string field names
 	"Max": allowField,
@@ -462,30 +461,6 @@ var callInfoByFunc = map[string]callInfo{
 	},
 }
 
-// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good
-// arg validation.
-func RegisterPluginFuncs(ops []ext.BitmapOp) {
-	for _, op := range ops {
-		// ignore overlap for now. This should change.
-		if _, ok := callInfoByFunc[op.Name]; ok {
-			continue
-		}
-		ci := callInfo{allowUnknown: true}
-		if len(op.Reserved) > 0 {
-			// mark these as valid/known reserved words
-			ci.prototypes = make(map[string]interface{})
-			for _, res := range op.Reserved {
-				ci.prototypes[res] = nil
-			}
-		}
-		t := op.Func.BitmapOpType()
-		if t.Precall == ext.OpPrecallGlobal {
-			ci.callType = PrecallGlobal
-		}
-		callInfoByFunc[op.Name] = ci
-	}
-}
-
 // CheckCallInfo tries to validate that arguments are correct and valid for the
 // given call. It does not guarantee checking all possible errors; for instance,
 // if an argument is a field name, CheckCallInfo can't validate that the field
@@ -505,6 +480,11 @@ func (c *Call) CheckCallInfo() error {
 		if !ok && strings.HasPrefix(k, "_") {
 			return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k)
 		}
+		if call, ok := v.(*Call); ok {
+			if err := call.CheckCallInfo(); err != nil {
+				return err
+			}
+		}
 		if acceptable == nil {
 			continue
 		}
diff --git a/rbf/db.go b/rbf/db.go
index 09f4b5aa9..3bb960f71 100644
--- a/rbf/db.go
+++ b/rbf/db.go
@@ -71,6 +71,10 @@ func NewDBWithShard(path string, shard int) *DB {
 	}
 }
 
+func (db *DB) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
+	panic("TODO: implement rbf.DB.DeleteFragment")
+}
+
 // DataPath returns the path to the data file for the DB.
 func (db *DB) DataPath() string {
 	return filepath.Join(db.Path, "data")
diff --git a/roaring/container_stash.go b/roaring/container_stash.go
index 3531a0317..095f462e5 100644
--- a/roaring/container_stash.go
+++ b/roaring/container_stash.go
@@ -387,7 +387,7 @@ func (c *Container) bitmap() []uint64 {
 // is provided. The target should be zeroed, or this becomes an implicit
 // union.
 func (c *Container) AsBitmap(target []uint64) (out []uint64) {
-	if c.typeID == containerBitmap {
+	if c != nil && c.typeID == containerBitmap {
 		return c.bitmap()
 	}
 	// Reminder: len(nil) == 0.
@@ -399,6 +399,10 @@ func (c *Container) AsBitmap(target []uint64) (out []uint64) {
 			out[i] = 0
 		}
 	}
+	// A nil *Container is a valid empty container.
+	if c == nil {
+		return out
+	}
 	if c.typeID == containerArray {
 		a := c.array()
 		for _, v := range a {
diff --git a/row.go b/row.go
index 30c11d2fe..ac0ef56e3 100644
--- a/row.go
+++ b/row.go
@@ -18,7 +18,6 @@ import (
 	"encoding/json"
 	"sort"
 
-	"github.com/molecula/ext"
 	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
@@ -326,65 +325,6 @@ func (r *Row) Union(others ...*Row) *Row {
 	return &Row{segments: output}
 }
 
-// GenericBinaryOp returns the output of a generic op on r and other.
-func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row {
-	var segments []rowSegment
-	itr := newMergeSegmentIterator(r.segments, other.segments)
-	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		if s1 == nil {
-			segments = append(segments, *s0)
-			continue
-		} else if s0 == nil {
-			segments = append(segments, *s1)
-			continue
-		}
-		segments = append(segments, *s0.GenericBinaryOp(op, s1, args))
-	}
-
-	return &Row{segments: segments}
-}
-
-// GenericNaryOp returns the output of an nary op on r and others.
-func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row {
-	segments := make([][]rowSegment, 0, len(others)+1)
-	if len(r.segments) > 0 {
-		segments = append(segments, r.segments)
-	}
-	nextSegs := make([][]rowSegment, 0, len(others)+1)
-	toProcess := make([]*rowSegment, 0, len(others)+1)
-	var output []rowSegment
-	for _, other := range others {
-		if len(other.segments) > 0 {
-			segments = append(segments, other.segments)
-		}
-	}
-	for len(segments) > 0 {
-		shard := segments[0][0].shard
-		for _, segs := range segments {
-			if segs[0].shard < shard {
-				shard = segs[0].shard
-			}
-		}
-		nextSegs = nextSegs[:0]
-		toProcess := toProcess[:0]
-		for _, segs := range segments {
-			if segs[0].shard == shard {
-				toProcess = append(toProcess, &segs[0])
-				segs = segs[1:]
-			}
-			if len(segs) > 0 {
-				nextSegs = append(nextSegs, segs)
-			}
-		}
-		// at this point, "toProcess" is a list of all the segments
-		// sharing the lowest ID, and nextSegs is a list of all the others.
-		// Swap the segment lists (so we don't have to reallocate it)
-		segments, nextSegs = nextSegs, segments
-		output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args))
-	}
-	return &Row{segments: output}
-}
-
 // Difference returns the diff of r and other.
 func (r *Row) Difference(others ...*Row) *Row {
 	var output []rowSegment
@@ -408,17 +348,6 @@ func (r *Row) Difference(others ...*Row) *Row {
 	return &Row{segments: output}
 }
 
-// GenericUnaryOp returns the results of a generic op on r.
-func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row {
-	work := r
-	var segments []rowSegment
-	for _, segment := range work.segments {
-		opped := segment.GenericUnaryOp(op, args)
-		segments = append(segments, *opped)
-	}
-	return &Row{segments: segments}
-}
-
 // Shift returns the bitwise shift of r by n bits.
 // Currently only positive shift values are supported.
 //
@@ -523,15 +452,6 @@ func (r *Row) Count() uint64 {
 	return n
 }
 
-// GenericCount applies an op to lots of things.
-func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 {
-	var n int64
-	for i := range r.segments {
-		n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args)
-	}
-	return uint64(n)
-}
-
 // MarshalJSON returns a JSON-encoded byte slice of r.
 func (r *Row) MarshalJSON() ([]byte, error) {
 	var o struct {
@@ -644,33 +564,6 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment {
 	}
 }
 
-// GenericOp performs a generic op on s and other
-func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment {
-	data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args)
-
-	return &rowSegment{
-		data:  UnwrapBitmap(data),
-		shard: s.shard,
-		n:     data.Count(),
-	}
-}
-
-// GenericOp performs a generic op on s and others
-func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment {
-	bitmaps := make([]ext.Bitmap, len(others)+1)
-	bitmaps[0] = WrapBitmap(s.data)
-	for i, seg := range others {
-		bitmaps[i+1] = WrapBitmap(seg.data)
-	}
-	data := op(bitmaps, args)
-
-	return &rowSegment{
-		data:  UnwrapBitmap(data),
-		shard: s.shard,
-		n:     data.Count(),
-	}
-}
-
 // Difference returns the diff of s and other.
 func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment {
 	datas := make([]*roaring.Bitmap, len(others))
@@ -713,17 +606,6 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
 	}, nil
 }
 
-// GenericUnaryOp returns s subject to op.
-func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment {
-	data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args))
-
-	return &rowSegment{
-		data:  data,
-		shard: s.shard,
-		n:     data.Count(),
-	}
-}
-
 // SetBit sets the i-th column of the row.
 func (s *rowSegment) SetBit(i uint64) (changed bool) {
 	s.ensureWritable()
diff --git a/server.go b/server.go
index f0974c0d5..a03a76899 100644
--- a/server.go
+++ b/server.go
@@ -27,13 +27,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/molecula/ext"
 	uuid "github.com/satori/go.uuid"
 
 	// extensions pulls in some extensions depending on build tags
 	_ "github.com/pilosa/pilosa/v2/extensions"
 	"github.com/pilosa/pilosa/v2/logger"
-	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pkg/errors"
@@ -63,7 +61,6 @@ type Server struct { // nolint: maligned
 	hosts            []string
 	clusterDisabled  bool
 	serializer       Serializer
-	extensions       []*ext.ExtensionInfo
 
 	// External
 	systemInfo    SystemInfo
@@ -430,30 +427,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	s.cluster.confirmDownRetries = s.confirmDownRetries
 	s.cluster.confirmDownSleep = s.confirmDownSleep
 	s.holder.broadcaster = s
-	err = s.loadAllExtensions()
-	if err != nil {
-		s.logger.Printf("not all plugins loaded successfully")
-	}
-	if len(s.extensions) > 0 {
-		s.logger.Printf("loaded extensions:")
-		for _, ext := range s.extensions {
-			if ext == nil {
-				s.logger.Printf("  inexplicably, a nil extension?!?")
-				continue
-			}
-			s.logger.Printf("  %s %s: %s", ext.Name, ext.Version, ext.Description)
-			if ext.License != "" {
-				s.logger.Printf("    License: %s", ext.License)
-			}
-			if len(ext.BitmapOps) > 0 {
-				opList := make([]string, len(ext.BitmapOps))
-				for i := range ext.BitmapOps {
-					opList[i] = ext.BitmapOps[i].Name
-				}
-				s.logger.Printf("    Ops: %s", strings.Join(opList, ", "))
-			}
-		}
-	}
 
 	err = s.cluster.setup()
 	if err != nil {
@@ -467,56 +440,6 @@ func (s *Server) InternalClient() InternalClient {
 	return s.defaultClient
 }
 
-// loadNewExtensions loads extensions that have been
-// registered since the last call to loadNewExtensions.
-func (s *Server) loadNewExtensions() error { //nolint:unused
-	return s.loadExtensions(ext.NewExtensions())
-}
-
-// loadAllExtensions loads all extensions.
-func (s *Server) loadAllExtensions() error {
-	return s.loadExtensions(ext.AllExtensions())
-}
-
-func (s *Server) loadExtensions(exts []*ext.ExtensionInfo) error {
-	var lastError error
-	for _, extension := range exts {
-		if err := s.loadExtension(extension); err != nil {
-			lastError = err
-		}
-	}
-	return lastError
-}
-
-func (s *Server) loadExtension(extInfo *ext.ExtensionInfo) error {
-	if extInfo.ExtensionAPI != "v0" {
-		return fmt.Errorf("%s: unsupported extension API %s", extInfo.Name, extInfo.ExtensionAPI)
-	}
-	s.extensions = append(s.extensions, extInfo)
-	bitmapOps := extInfo.BitmapOps
-	bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0
-	for i := range bitmapOps {
-		typ := bitmapOps[i].Func.BitmapOpType()
-		switch {
-		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
-			countOps++
-		case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
-			bmOps++
-		case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
-			fieldOps++
-		default:
-			unknownOps++
-		}
-	}
-	err := s.executor.registerOps(bitmapOps)
-	if err != nil {
-		s.logger.Printf("warning: extension registration failed: %v", err)
-	} else {
-		pql.RegisterPluginFuncs(bitmapOps)
-	}
-	return nil
-}
-
 // UpAndDown brings the server up minimally and shuts it down
 // again; basically, it exists for testing holder open and close.
 func (s *Server) UpAndDown() error {
@@ -545,8 +468,12 @@ func (s *Server) UpAndDown() error {
 func (s *Server) Open() error {
 	s.logger.Printf("open server")
 
-	// Start background monitoring.
-	s.snapshotQueue = newSnapshotQueue(10, 2, s.logger)
+	if s.holder.NeedsSnapshot() {
+		// Start background monitoring.
+		s.snapshotQueue = newSnapshotQueue(10, 2, s.logger)
+	} else {
+		s.snapshotQueue = defaultSnapshotQueue //TODO (twg) rethink this
+	}
 
 	// Log startup
 	err := s.holder.logStartup()
@@ -711,6 +638,7 @@ func (s *Server) monitorAntiEntropy() {
 			// the cluster sets its state to resizing and *then* sends to
 			// abortAntiEntropyCh before starting to resize
 		}
+
 		// Sync holders.
 		s.logger.Printf("holder sync beginning")
 		s.cluster.muAntiEntropy.Lock()
diff --git a/test/index.go b/test/index.go
index b90702ae2..e51850d87 100644
--- a/test/index.go
+++ b/test/index.go
@@ -32,7 +32,9 @@ func newIndex() *Index {
 	if err != nil {
 		panic(err)
 	}
-	index, err := pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i")
+	h := pilosa.NewHolder(pilosa.DefaultPartitionN)
+	h.Path = path
+	index, err := h.CreateIndex("i", pilosa.IndexOptions{})
 	if err != nil {
 		panic(err)
 	}
@@ -42,7 +44,7 @@ func newIndex() *Index {
 // MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
 func MustOpenIndex() *Index {
 	index := newIndex()
-	if err := index.Open(); err != nil {
+	if err := index.Open(false); err != nil {
 		panic(err)
 	}
 	return index
@@ -62,12 +64,14 @@ func (i *Index) Reopen() error {
 	}
 
 	path, name := i.Path(), i.Name()
-	i.Index, err = pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, name)
+	h := pilosa.NewHolder(pilosa.DefaultPartitionN)
+	h.Path = h.HolderPathFromIndexPath(path, name)
+	i.Index, err = h.CreateIndex(name, pilosa.IndexOptions{})
 	if err != nil {
 		return err
 	}
 
-	if err := i.Open(); err != nil {
+	if err := i.Open(false); err != nil {
 		return err
 	}
 	return nil
diff --git a/tx.go b/tx.go
index 9fcf6fd99..1dbea6daf 100644
--- a/tx.go
+++ b/tx.go
@@ -15,7 +15,12 @@
 package pilosa
 
 import (
+	"bytes"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
 	"sync"
 
 	"github.com/pilosa/pilosa/v2/roaring"
@@ -50,6 +55,10 @@ const writable = true
 // that have not been committed.
 type Tx interface {
 
+	// Type returns "roaring", "rbf", "badger", "badger_roaring", or one of the other
+	// blue-green Tx types at the top of txfactory.go
+	Type() string
+
 	// Rollback must be called the end of read-only transactions. Either
 	// Rollback or Commit must be called at the end of writable transactions.
 	// It is safe to call Rollback multiple times, but it must be
@@ -97,6 +106,11 @@ type Tx interface {
 	// Calling Next() on the returned roaring.ContainerIterator gives
 	// you a roaring.Container that is either run, array, or raw bitmap.
 	// Return value 'found' is true when the ckey container was present.
+	// ckey of 0 gives all containers (in the fragment).
+	//
+	// ContainerIterator must not have side-effects. blueGreenTx will
+	// call it at the very beginning of commit to verify db contents.
+	//
 	ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error)
 
 	// RoaringBitmap retreives the roaring.Bitmap for the entire shard.
@@ -162,10 +176,52 @@ type Tx interface {
 	OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error)
 
 	// ImportRoaringBits does efficient bulk import using rit, a roaring.RoaringIterator.
+	//
 	// See the roaring package for details of the RoaringIterator.
+	//
 	// If clear is true, the bits from rit are cleared, otherwise they are set in the
 	// specifed fragment.
-	ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error)
+	//
+	// The data argument can be nil, its ignored for RBF/BadgerTx. It is supplied to
+	// RoaringTx.ImportRoaringBits() in fragment.go fragment.fillFragmentFromArchive()
+	// to do the traditional fragment.readStorageFromArchive() which
+	// does some in memory field/view/fragment metadata updates.
+	// It makes blueGreenTx testing viable too.
+	//
+	// ImportRoaringBits return values changed and rowSet may be inaccurate if
+	// the data []byte is supplied (the RoaringTx implementation neglects this for speed).
+	ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error)
+
+	RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error)
+
+	// SliceOfShards returns all of the shards for the specified index, field, view triple.
+	// Use within pilosa supposes a new read-only transaction was created just
+	// for the SliceOfShards() call. The original Roaring version is the only
+	// one that needs optionalViewPath; any other Tx implementation can ignore that.
+	SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error)
+}
+
+// TxStore has operations that will create and commit multiple
+// Tx on a backing store.
+type TxStore interface {
+
+	// DeleteFragment deletes all the containers in a fragment.
+	//
+	// This is not in a Tx because it will often do too many deletes for a single
+	// transaction, and clients would be suprised to find their Tx had already
+	// been commited and they are getting an error on double-Commit.
+	// Instead each TxStore implementation creates and commits as many
+	// transactions as needed.
+	//
+	// Argument frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
+	// If not nil, it must be of type *fragment. If frag is supplied, then
+	// index must be equal to frag.index, field equal to frag.field, view equal
+	// to frag.view, and shard equal to frag.shard.
+	//
+	DeleteFragment(index, field, view string, shard uint64, frag interface{}) error
+
+	// Close shuts down the database.
+	Close() error
 }
 
 // RawRoaringData used by ImportRoaringBits.
@@ -207,10 +263,26 @@ func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx {
 
 var _ Tx = (*MultiTx)(nil)
 
+func (mtx *MultiTx) Type() string {
+	return RoaringTxn
+}
+
+func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
+	tx, err := mtx.txNoShard(index)
+	panicOn(err)
+	return tx.SliceOfShards(index, field, view, optionalViewPath)
+}
+
 func (mtx *MultiTx) UseRowCache() bool {
 	return true
 }
 
+func (mtx *MultiTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+	tx, err := mtx.tx(index, shard)
+	panicOn(err)
+	return tx.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
+}
+
 func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
 	tx, err := mtx.tx(index, shard)
 	panicOn(err)
@@ -226,8 +298,10 @@ func (mtx *MultiTx) Pointer() string {
 	return fmt.Sprintf("%p", mtx)
 }
 
-func (tx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
-	panic("not done")
+func (mtx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
+	tx, err := mtx.tx(index, shard)
+	panicOn(err)
+	return tx.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
 }
 
 func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
@@ -412,6 +486,20 @@ func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
 	return tx, nil
 }
 
+// version of the above for SliceOfShards(), where we don't have a shard.
+func (mtx *MultiTx) txNoShard(index string) (_ Tx, err error) {
+	mtx.mu.Lock()
+	defer mtx.mu.Unlock()
+
+	// Lookup transaction from cache.
+	for _, tx := range mtx.txs {
+		if tx.(*RoaringTx).Index.name == index {
+			return tx, nil
+		}
+	}
+	panic(fmt.Sprintf("no prior RoaringTx available, looking up index='%v'", index))
+}
+
 type multiTxKey struct {
 	index string
 	shard uint64
@@ -426,10 +514,46 @@ type RoaringTx struct {
 	fragment *fragment
 }
 
+func (mtx *RoaringTx) Type() string {
+	return RoaringTxn
+}
+
 func (tx *RoaringTx) UseRowCache() bool {
 	return true
 }
 
+func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
+
+	// SliceOfShards is based on view.openFragments()
+
+	file, err := os.Open(filepath.Join(optionalViewPath, "fragments"))
+	if os.IsNotExist(err) {
+		return
+	} else if err != nil {
+		return nil, errors.Wrap(err, "opening fragments directory")
+	}
+	defer file.Close()
+
+	fis, err := file.Readdir(0)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading fragments directory")
+	}
+
+	for _, fi := range fis {
+		if fi.IsDir() {
+			continue
+		}
+		// Parse filename into integer.
+		shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
+		if err != nil {
+			//v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
+			continue
+		}
+		sliceOfShards = append(sliceOfShards, shard)
+	}
+	return
+}
+
 func (tx *RoaringTx) Pointer() string {
 	return fmt.Sprintf("%p", tx)
 }
@@ -442,13 +566,25 @@ func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roa
 	return b.Iterator()
 }
 
-func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
-	b, err := tx.bitmap(index, field, view, shard)
-	panicOn(err)
+// ImportRoaringBits return values changed and rowSet will be inaccurate if
+// the data []byte is supplied. This mimics the traditional roaring-per-file
+// and should be faster.
+func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
+
+	f, err := tx.getFragment(index, field, view, shard)
 	if err != nil {
 		return 0, nil, err
 	}
-	return b.ImportRoaringRawIterator(rit, clear, true, rowSize)
+	if len(data) > 0 {
+		// changed and rowSet are ignored anyway when len(data) > 0;
+		// when we are called from fragment.fillFragmentFromArchive()
+		// which is the only place the data []byte is supplied.
+		// blueGreenTx also turns off the checks in this case.
+		return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data))
+	}
+
+	changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize)
+	return
 }
 
 func (tx *RoaringTx) Readonly() bool {
@@ -625,8 +761,16 @@ func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset
 // getFragment is used by IncrementOpN() and by bitmap()
 func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*fragment, error) {
 
-	// If a fragment is attached, always use it.
+	// If a fragment is attached, always use it. Since it was set at Tx creation,
+	// it is highly likely to be correct.
 	if tx.fragment != nil {
+		// but still a basic sanity check.
+		if tx.fragment.index != index ||
+			tx.fragment.field != field ||
+			tx.fragment.view != view ||
+			tx.fragment.shard != shard {
+			panic(fmt.Sprintf("different fragment cached vs requested. tx.fragment='%#v', index='%v', field='%v'; view='%v'; shard='%v'", tx.fragment, index, field, view, shard))
+		}
 		return tx.fragment, nil
 	}
 
@@ -659,8 +803,10 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag
 	}
 
 	frag := v.Fragment(shard)
+
 	if frag == nil {
-		panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard))
+		return nil, fmt.Errorf("fragment not found: %q / %q / %d", field, view, shard)
+		//panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard))
 	}
 
 	// Note: we cannot cache frag into tx.fragment.
@@ -677,3 +823,52 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B
 	}
 	return frag.storage, nil
 }
+
+type RoaringStore struct{}
+
+func NewRoaringStore() *RoaringStore {
+	return &RoaringStore{}
+}
+
+func (db *RoaringStore) Close() error {
+	return nil
+}
+
+// frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
+func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
+
+	fragment, ok := frag.(*fragment)
+	if !ok {
+		return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
+	}
+
+	// Close data files before deletion.
+	if err := fragment.Close(); err != nil {
+		return errors.Wrap(err, "closing fragment")
+	}
+
+	// Delete fragment file.
+	if err := os.Remove(fragment.path); err != nil {
+		return errors.Wrap(err, "deleting fragment file")
+	}
+
+	// Delete fragment cache file.
+	if err := os.Remove(fragment.cachePath()); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard))
+	}
+	return nil
+}
+
+func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+	file, err := os.Open(fragmentPathForRoaring) // open the fragment file
+	if err != nil {
+		return nil, -1, err
+	}
+	fi, err := file.Stat()
+	if err != nil {
+		return nil, -1, errors.Wrap(err, "statting")
+	}
+	sz = fi.Size()
+	r = file
+	return
+}
diff --git a/txfactory.go b/txfactory.go
index f8af55c6c..0f49982ec 100644
--- a/txfactory.go
+++ b/txfactory.go
@@ -22,6 +22,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/pilosa/pilosa/v2/rbf"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -55,15 +56,34 @@ var sep = string(os.PathSeparator)
 type TxFactory struct {
 	typeOfTx txtype
 
-	bw *BadgerDBWrapper
+	badgerDB *BadgerDBWrapper
+
+	rbfDB *rbf.DB
+
+	roaringDB *RoaringStore
 
 	// could have more than one *Index, but for now keep it simple,
 	// and allow blueGreenTx to report badger contents via idx
 	idx *Index
-
-	// TODO:  put RBF database handle here.
 }
 
+/* want glue-green to multiplex, so don't do this directly
+// but rather f.CloseStore()
+func (f *TxFactory) Store() TxStore {
+	switch f.typeOfTx {
+	case roaringFragmentFilesTxn:
+		return &RoaringStore{}
+	case badgerTxn:
+		return f.badgerDB
+	case rbfTxn:
+		return f.rbfDB
+		//	case blueGreenBadgerRoaring:
+		//	case blueGreenRoaringBadger:
+	}
+	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
+}
+*/
+
 // integer types for fast switch{}
 type txtype int
 
@@ -85,6 +105,32 @@ const (
 	blueGreenRBFBadger txtype = 9
 )
 
+func (txf *TxFactory) NeedsSnapshot() bool {
+	switch txf.typeOfTx {
+	case noneTxn:
+		panic("noneTxn should not occur")
+	case roaringFragmentFilesTxn:
+		return true
+	case badgerTxn:
+		return false
+	case rbfTxn:
+		return false
+	case blueGreenBadgerRoaring:
+		return true
+	case blueGreenRoaringBadger:
+		return true
+	case blueGreenRBFRoaring:
+		return true
+	case blueGreenRoaringRBF:
+		return true
+	case blueGreenBadgerRBF:
+		return false
+	case blueGreenRBFBadger:
+		return false
+	}
+	panic(fmt.Sprintf("unknown typeOfTx '%v'", txf.typeOfTx))
+}
+
 func MustTxsrcToTxtype(txsrc string) txtype {
 	switch txsrc {
 	case RoaringTxn: // "roaring"
@@ -109,29 +155,52 @@ func MustTxsrcToTxtype(txsrc string) txtype {
 	panic(fmt.Sprintf("unknown txsrc '%v'", txsrc))
 }
 
-func newTxFactory(txsrc string, path string) (f *TxFactory, err error) {
+// always store files in a subdir of dir. If we are having one
+// database or many can depend on name.
+func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
+
 	ty := MustTxsrcToTxtype(txsrc)
 	if ty < 1 || ty > 9 {
 		panic(fmt.Sprintf("invalid txtype '%v'", int(ty)))
 	}
 
-	var bw *BadgerDBWrapper
-	if ty == badgerTxn || ty == 4 || ty == 5 || ty == 8 || ty == 9 {
-		bw, err = openBadgerDBWrapper(path)
+	f = &TxFactory{
+		typeOfTx:  ty,
+		roaringDB: NewRoaringStore(),
+	}
+	switch ty {
+	case badgerTxn, blueGreenBadgerRoaring, blueGreenRoaringBadger, blueGreenBadgerRBF, blueGreenRBFBadger:
+
+		// one, big, bad-ass badger for all data: the honeyBadger.
+		//
+		// Note that having a single Tx backing store for all indexes
+		// enables cross-index Tx, which are important and are tested for.
+		path := dir + sep + "honeyBadger"
+
+		f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
 		// TODO(jea): figure out what the appropriate error path is here.
 		//fmt.Printf("warning: could not open badgerdb on path '%v': '%v'. For safety, we are opening a new '%v-fallback' instead\n", path, err, path+"-fallback")
 		if err != nil {
-			//bw, err = newBadgerDBWrapper(path + "-fallback")
-			bw, err = newBadgerDBWrapper(path)
+			f.badgerDB, err = globalBadgerReg.newBadgerDBWrapper(path)
 		}
-		panicOn(err)
-
-		bw.doAllocZero = true
+		if err != nil {
+			return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
+		}
+		// electric-fence like finding of access to mmapped data beyond
+		// transaction end time.
+		f.badgerDB.doAllocZero = true
 	}
-	return &TxFactory{
-		typeOfTx: ty,
-		bw:       bw,
-	}, err
+
+	switch ty {
+	case rbfTxn, blueGreenRBFRoaring, blueGreenRoaringRBF, blueGreenBadgerRBF, blueGreenRBFBadger:
+		path := dir + sep + name + ".rbf"
+		f.rbfDB = rbf.NewDB(path)
+		if err := f.rbfDB.Open(); err != nil {
+			return nil, errors.Wrap(err, fmt.Sprintf("cannot open rbf db. path='%v'", path))
+		}
+	}
+
+	return f, err
 }
 
 // Txo holds the transaction options
@@ -153,34 +222,35 @@ func (f *TxFactory) DeleteIndex(name string) error {
 		// from holder.go:955, by default is already done there with os.RemoveAll()
 		return nil
 	case badgerTxn:
-		return f.bw.DeleteIndex(name)
+		return f.badgerDB.DeleteIndex(name)
 	case rbfTxn:
 		panic("todo rbfTxn DeleteIndex(name)")
 	case blueGreenBadgerRoaring:
-		return f.bw.DeleteIndex(name)
+		return f.badgerDB.DeleteIndex(name)
 	case blueGreenRoaringBadger:
-		return f.bw.DeleteIndex(name)
+		return f.badgerDB.DeleteIndex(name)
 	}
 	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
 }
 
-func (f *TxFactory) Close() error {
+func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uint64, frag *fragment) error {
 	switch f.typeOfTx {
 	case roaringFragmentFilesTxn:
-		return nil
+		return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
 	case badgerTxn:
-		// note cannot actually close Badger here.
-		// causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed.
-		//return f.bw.Close()
-		return nil
+		return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
 	case rbfTxn:
-		panic("todo rbfTxn Close()")
+		//return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
+		return nil
 	case blueGreenBadgerRoaring:
-		return nil
+		_ = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
+		return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
 	case blueGreenRoaringBadger:
-		return nil
+		_ = f.roaringDB.DeleteFragment(index, field, view, shard, frag)
+		return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
 	}
 	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
+
 }
 
 func (f *TxFactory) CloseIndex(idx *Index) error {
@@ -188,10 +258,14 @@ func (f *TxFactory) CloseIndex(idx *Index) error {
 	case roaringFragmentFilesTxn:
 		return nil
 	case badgerTxn:
+		// note cannot actually close Badger here.
+		// causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed.
+		//return f.badgerDB.Close()
 		return nil
 	case rbfTxn:
-		panic("todo rbfTxn CloseIndex()")
-
+		// for same reason as above may not be able to close here.
+		//return f.rbfDB.Close()
+		return nil
 	case blueGreenBadgerRoaring:
 		return nil
 	case blueGreenRoaringBadger:
@@ -202,23 +276,66 @@ func (f *TxFactory) CloseIndex(idx *Index) error {
 
 func (f *TxFactory) NewTx(o Txo) Tx {
 
+	indexName := ""
+	if o.Index != nil {
+		indexName = o.Index.name
+	}
+
 	switch f.typeOfTx {
 	case roaringFragmentFilesTxn:
 		return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
 	case badgerTxn:
-		btx := f.bw.NewBadgerTx(o.Write)
+		btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
 		return btx
 	case rbfTxn:
 		panic("todo rbfTxn creation")
-
+		/*
+			rbftx, err := f.rbfDB.Begin(o.Write)
+			if err != nil {
+				errors.Wrap(err, "rbfDB.Begin transaction errored")
+			}
+			return rbftx
+		*/
 	case blueGreenBadgerRoaring:
-		btx := f.bw.NewBadgerTx(o.Write)
+		btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
 		rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
 		return newBlueGreenTx(btx, rtx, f.idx)
 	case blueGreenRoaringBadger:
-		btx := f.bw.NewBadgerTx(o.Write)
+		btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
 		rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
 		return newBlueGreenTx(rtx, btx, f.idx)
+
+		/*
+			case blueGreenBadgerRBF:
+				btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
+				rbftx, err := f.rbfDB.Begin(o.Write)
+				if err != nil {
+					errors.Wrap(err, "rbfDB.Begin transaction errored")
+				}
+				return newBlueGreenTx(btx, rbftx, f.idx)
+			case blueGreenRBFBadger:
+				btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
+				rbftx, err := f.rbfDB.Begin(o.Write)
+				if err != nil {
+					errors.Wrap(err, "rbfDB.Begin transaction errored")
+				}
+				return newBlueGreenTx(rbftx, btx, f.idx)
+
+			case blueGreenRBFRoaring:
+				rbftx, err := f.rbfDB.Begin(o.Write)
+				if err != nil {
+					errors.Wrap(err, "rbfDB.Begin transaction errored")
+				}
+				rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
+				return newBlueGreenTx(rbftx, rtx, f.idx)
+			case blueGreenRoaringRBF:
+				rbftx, err := f.rbfDB.Begin(o.Write)
+				if err != nil {
+					errors.Wrap(err, "rbfDB.Begin transaction errored")
+				}
+				rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
+				return newBlueGreenTx(rtx, rbftx, f.idx)
+		*/
 	}
 	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
 }
@@ -255,7 +372,7 @@ func (ty txtype) String() string {
 // Hence to view uncommited keys, you must provide in optionalUseThisTx the
 // Tx in which they have been added.
 func (idx *Index) StringifiedBadgerKeys(optionalUseThisTx Tx) string {
-	return idx.Txf.bw.StringifiedBadgerKeys(optionalUseThisTx)
+	return idx.Txf.badgerDB.StringifiedBadgerKeys(optionalUseThisTx)
 }
 
 // fragmentSpecFromRoaringPath takes a path releative to the
diff --git a/utils_internal_test.go b/utils_internal_test.go
index e9603b222..d87499a88 100644
--- a/utils_internal_test.go
+++ b/utils_internal_test.go
@@ -15,8 +15,6 @@
 package pilosa
 
 import (
-	"bufio"
-	"bytes"
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
@@ -396,9 +394,8 @@ func (b bcast) SendTo(to *Node, m Message) error {
 	return nil
 }
 
-// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
+// FollowResizeInstruction is a version of cluster.followResizeInstruction used for testing.
 func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error {
-
 	// Prepare the return message.
 	complete := &ResizeInstructionComplete{
 		JobID: instr.JobID,
@@ -420,7 +417,8 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
 		}
 
 		// Sync available shards.
-		for _, is := range instr.NodeStatus.Indexes {
+		for k, is := range instr.NodeStatus.Indexes {
+			_ = k
 			for _, fs := range is.Fields {
 				f := destCluster.holder.Field(is.Name, fs.Name)
 
@@ -451,23 +449,28 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
 				}
 			}
 
-			buf := bytes.NewBuffer(nil)
+			// this is the *test* version of a network call, transferring fragments between
+			// nodes in a cluster. So it is allowed to be kind of a hack.
 
-			bw := bufio.NewWriter(buf)
-			br := bufio.NewReader(buf)
+			// there will be two -badgerdb directories/databases, we need to copy
+			// from src to dest the fragment. This simulates sending the fragment over the network.
+			srcIdx := srcCluster.holder.Index(src.Index)
+			srctx := srcIdx.Txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment})
 
-			// Get the fragment from source.
-			if _, err := srcFragment.WriteTo(bw); err != nil {
-				return err
-			}
-
-			// Flush the bufio.buf to the io.Writer (buf).
-			bw.Flush()
-
-			// Write data to destination.
-			if _, err := destFragment.ReadFrom(br); err != nil {
-				return err
+			destIdx := destCluster.holder.Index(src.Index)
+			desttx := destIdx.Txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment})
+
+			citer, _, err := srctx.ContainerIterator(src.Index, src.Field, src.View, src.Shard, 0)
+			panicOn(err)
+			d := destFragment
+			for citer.Next() {
+				ckey, c := citer.Value()
+				err := desttx.PutContainer(d.index, d.field, d.view, d.shard, ckey, c)
+				panicOn(err)
 			}
+			citer.Close()
+			panicOn(desttx.Commit())
+			srctx.Rollback()
 		}
 
 		return nil
diff --git a/view.go b/view.go
index 12210a619..106ae2b42 100644
--- a/view.go
+++ b/view.go
@@ -50,6 +50,7 @@ type view struct {
 	qualifiedName string
 
 	holder *Holder
+	idx    *Index
 
 	fieldType string
 	cacheType string
@@ -143,7 +144,8 @@ func (v *view) open() error {
 		}
 
 		v.holder.Logger.Debugf("open fragments for index/field/view: %s/%s/%s", v.index, v.field, v.name)
-		if err := v.openFragments(); err != nil {
+
+		if err := v.openFragmentsInTx(); err != nil {
 			return errors.Wrap(err, "opening fragments")
 		}
 
@@ -159,64 +161,27 @@ func (v *view) open() error {
 
 var workQueue = make(chan struct{}, runtime.NumCPU()*2)
 
-// openFragments opens and initializes the fragments inside the view.
-func (v *view) openFragments() error {
-	file, err := os.Open(filepath.Join(v.path, "fragments"))
-	if os.IsNotExist(err) {
-		return nil
-	} else if err != nil {
-		return errors.Wrap(err, "opening fragments directory")
-	}
-	defer file.Close()
+// replaces v.openFragments() with Tx generic code.
+func (v *view) openFragmentsInTx() error {
 
-	fis, err := file.Readdir(0)
+	tx := v.idx.Txf.NewTx(Txo{Write: !writable, Index: v.idx})
+	defer tx.Rollback()
+
+	shards, err := tx.SliceOfShards(v.index, v.field, v.name, v.path)
 	if err != nil {
-		return errors.Wrap(err, "reading fragments directory")
+		return errors.Wrap(err, "SliceOfShards")
 	}
-
-	eg, ctx := errgroup.WithContext(context.Background())
-	var mu sync.Mutex
-
-fileLoop:
-	for _, loopFi := range fis {
-		select {
-		case <-ctx.Done():
-			break fileLoop
-		default:
-			fi := loopFi
-
-			if fi.IsDir() {
-				continue
-			}
-
-			// Parse filename into integer.
-			shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
-			if err != nil {
-				v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
-				continue
-			}
-
-			workQueue <- struct{}{}
-			v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard)
-			eg.Go(func() error {
-				defer func() {
-					<-workQueue
-				}()
-				frag := v.newFragment(v.fragmentPath(shard), shard)
-				if err := frag.Open(); err != nil {
-					return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
-				}
-				frag.RowAttrStore = v.rowAttrStore
-				v.holder.Logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard)
-				mu.Lock()
-				v.fragments[frag.shard] = frag
-				v.addKnownShard(frag.shard)
-				mu.Unlock()
-				return nil
-			})
+	for _, shard := range shards {
+		frag := v.newFragment(v.fragmentPath(shard), shard)
+		if err := frag.Open(); err != nil {
+			return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
 		}
+		frag.RowAttrStore = v.rowAttrStore
+		v.holder.Logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard)
+		v.fragments[frag.shard] = frag
+		v.addKnownShard(frag.shard)
 	}
-	return eg.Wait()
+	return nil
 }
 
 // close closes the view and its fragments.
@@ -359,6 +324,16 @@ func (v *view) notifyIfNewShard(shard uint64) {
 }
 
 func (v *view) newFragment(path string, shard uint64) *fragment {
+
+	if v.holder != nil && v.idx != nil {
+		// A view must have its v.idx *Index registered with its holder.
+		// Otherwise TestField_AvailableShards crashes, as one example.
+		hIdx := v.holder.Index(v.idx.name)
+		if hIdx == nil && v.idx != nil {
+			v.holder.addIndexFromField(v.idx)
+		}
+	}
+
 	frag := newFragment(v.holder, path, v.index, v.field, v.name, shard, v.flags())
 	frag.CacheType = v.cacheType
 	frag.CacheSize = v.cacheSize
@@ -375,28 +350,17 @@ func (v *view) newFragment(path string, shard uint64) *fragment {
 func (v *view) deleteFragment(shard uint64) error {
 	v.mu.Lock()
 	defer v.mu.Unlock()
-	fragment := v.fragments[shard]
-	if fragment == nil {
+	f := v.fragments[shard]
+	if f == nil {
 		return ErrFragmentNotFound
 	}
 
 	v.holder.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)
 
-	// Close data files before deletion.
-	if err := fragment.Close(); err != nil {
-		return errors.Wrap(err, "closing fragment")
+	idx := f.holder.Index(v.index)
+	if err := idx.Txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil {
+		return errors.Wrap(err, "DeleteFragment")
 	}
-
-	// Delete fragment file.
-	if err := os.Remove(fragment.path); err != nil {
-		return errors.Wrap(err, "deleting fragment file")
-	}
-
-	// Delete fragment cache file.
-	if err := os.Remove(fragment.cachePath()); err != nil {
-		v.holder.Logger.Printf("no cache file to delete for shard %d", shard)
-	}
-
 	delete(v.fragments, shard)
 	v.removeKnownShard(shard)
 
diff --git a/view_internal_test.go b/view_internal_test.go
index bbb1a20f4..bcde5f871 100644
--- a/view_internal_test.go
+++ b/view_internal_test.go
@@ -34,7 +34,15 @@ func mustOpenView(index, field, name string) *view {
 		CacheSize: DefaultCacheSize,
 	}
 
-	v := newView(NewHolder(DefaultPartitionN), path, index, field, name, fo)
+	h := NewHolder(DefaultPartitionN)
+	h.Path = path
+	// h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment
+	idx, err := h.createIndex(index, IndexOptions{})
+	_ = idx
+	panicOn(err)
+
+	v := newView(h, path, index, field, name, fo)
+	v.idx = idx
 	if err := v.open(); err != nil {
 		panic(err)
 	}
diff --git a/vprint.go b/vprint.go
index bc40dfe45..52130bbeb 100644
--- a/vprint.go
+++ b/vprint.go
@@ -111,3 +111,33 @@ func FileLine(depth int) string {
 func stack() string {
 	return string(debug.Stack())
 }
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}

From 38eea9b4a7cf8276afb23d5caa8751da5251cc84 Mon Sep 17 00:00:00 2001
From: Jason Aten <jason.aten@molecula.com>
Date: Mon, 27 Jul 2020 20:25:43 -0400
Subject: [PATCH 07/14] reparallelize view.go openFragmentsInTx()

---
 view.go | 37 +++++++++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/view.go b/view.go
index 106ae2b42..682b3fb00 100644
--- a/view.go
+++ b/view.go
@@ -171,17 +171,38 @@ func (v *view) openFragmentsInTx() error {
 	if err != nil {
 		return errors.Wrap(err, "SliceOfShards")
 	}
+
+	eg, ctx := errgroup.WithContext(context.Background())
+	var mu sync.Mutex
+
+shardLoop:
 	for _, shard := range shards {
-		frag := v.newFragment(v.fragmentPath(shard), shard)
-		if err := frag.Open(); err != nil {
-			return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
+		select {
+		case <-ctx.Done():
+			break shardLoop
+		default:
+
+			workQueue <- struct{}{}
+			v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard)
+			eg.Go(func() error {
+				defer func() {
+					<-workQueue
+				}()
+				frag := v.newFragment(v.fragmentPath(shard), shard)
+				if err := frag.Open(); err != nil {
+					return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
+				}
+				frag.RowAttrStore = v.rowAttrStore
+				v.holder.Logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard)
+				mu.Lock()
+				v.fragments[frag.shard] = frag
+				v.addKnownShard(frag.shard)
+				mu.Unlock()
+				return nil
+			})
 		}
-		frag.RowAttrStore = v.rowAttrStore
-		v.holder.Logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard)
-		v.fragments[frag.shard] = frag
-		v.addKnownShard(frag.shard)
 	}
-	return nil
+	return eg.Wait()
 }
 
 // close closes the view and its fragments.

From 71eccd121decefb884d08e87be042375102f4139 Mon Sep 17 00:00:00 2001
From: "Jason E. Aten" <jason.aten@molecula.com>
Date: Tue, 28 Jul 2020 07:50:22 -0400
Subject: [PATCH 08/14] fix race in view.openFragmentsInTx

---
 view.go | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/view.go b/view.go
index 682b3fb00..da61f1837 100644
--- a/view.go
+++ b/view.go
@@ -175,19 +175,32 @@ func (v *view) openFragmentsInTx() error {
 	eg, ctx := errgroup.WithContext(context.Background())
 	var mu sync.Mutex
 
+	shardCh := make(chan uint64, len(shards))
+	for i := range shards {
+		shardCh <- shards[i]
+	}
+
 shardLoop:
-	for _, shard := range shards {
+	for range shards {
 		select {
 		case <-ctx.Done():
 			break shardLoop
 		default:
 
 			workQueue <- struct{}{}
-			v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard)
 			eg.Go(func() error {
 				defer func() {
 					<-workQueue
 				}()
+
+				var shard uint64
+				select {
+				case shard = <-shardCh:
+				default:
+					return nil // no more work
+				}
+				v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard)
+
 				frag := v.newFragment(v.fragmentPath(shard), shard)
 				if err := frag.Open(); err != nil {
 					return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)

From 64de2081705dede197b2df8632f1688ac9e35b73 Mon Sep 17 00:00:00 2001
From: Ben Johnson <benbjohnson@yahoo.com>
Date: Wed, 29 Jul 2020 08:58:06 -0600
Subject: [PATCH 09/14] Implement pilosa.Tx for RBF

---
 api_test.go                      |   4 +
 badger.go                        |   1 -
 executor_test.go                 |   7 +-
 fragment.go                      |   4 +-
 fragment_internal_test.go        |  15 +
 holder_test.go                   |  11 +
 mmap_test.go                     |   2 +
 pilosa_test.go                   |   7 +
 rbf/cursor.go                    |  27 +-
 rbf/cursor_test.go               |  40 +--
 rbf/cursorx.go                   |   5 +-
 rbf/db.go                        |  15 +-
 rbf/db_test.go                   |   5 +-
 rbf/rbf.go                       |  69 +++-
 rbf/rbf_test.go                  |   8 -
 rbf/tx.go                        | 548 ++++++++++++++++++++++++++++---
 rbf/tx_test.go                   |  44 +--
 roaring/roaring.go               |  22 +-
 roaring/roaring_internal_test.go |  30 +-
 tx.go                            | 146 ++++++++
 txfactory.go                     |  40 +--
 xrbrsupport.go                   |   2 +-
 22 files changed, 882 insertions(+), 170 deletions(-)

diff --git a/api_test.go b/api_test.go
index 3942a7337..b52906514 100644
--- a/api_test.go
+++ b/api_test.go
@@ -165,6 +165,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
 }
 
 func TestAPI_Import(t *testing.T) {
+	skipForRBF(t)
+
 	c := test.MustRunCluster(t, 2,
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
@@ -276,6 +278,8 @@ func TestAPI_Import(t *testing.T) {
 }
 
 func TestAPI_ImportValue(t *testing.T) {
+	skipForRBF(t)
+
 	c := test.MustRunCluster(t, 2,
 		[]server.CommandOption{
 			server.OptCommandServerOptions(
diff --git a/badger.go b/badger.go
index aee8d9897..b426e630e 100644
--- a/badger.go
+++ b/badger.go
@@ -276,7 +276,6 @@ func badgerPath(path string) string {
 // the existing instance. This insures only one badgerDB
 // per bpath in this pilosa node.
 func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
-
 	// now that newTxFactory can call us directly, we might not
 	// have the -badgerdb suffix.
 	if !strings.HasSuffix(bpath, "-badgerdb") {
diff --git a/executor_test.go b/executor_test.go
index 73f1d44ce..b19c6cdf0 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -3624,6 +3624,8 @@ func TestExecutor_Execute_FieldValue(t *testing.T) {
 
 // Ensure an all query can be executed.
 func TestExecutor_Execute_All(t *testing.T) {
+	skipForRBF(t)
+
 	t.Run("ColumnID", func(t *testing.T) {
 		c := test.MustRunCluster(t, 1)
 		defer c.Close()
@@ -4026,7 +4028,6 @@ func TestExecutor_Execute_ClearRow(t *testing.T) {
 
 // Ensure a row can be set.
 func TestExecutor_Execute_SetRow(t *testing.T) {
-
 	t.Run("Set_NewRow", func(t *testing.T) {
 		c := test.MustRunCluster(t, 1)
 		defer c.Close()
@@ -4385,6 +4386,8 @@ func TestExecutor_Execute_Query_Error(t *testing.T) {
 }
 
 func TestExecutor_GroupByStrings(t *testing.T) {
+	skipForRBF(t)
+
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
 	c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "generals", pilosa.OptFieldKeys())
@@ -4841,6 +4844,8 @@ func sameStringSlice(x, y []string) bool {
 }
 
 func TestExecutor_Execute_GroupBy(t *testing.T) {
+	skipForRBF(t)
+
 	groupByTest := func(t *testing.T, clusterSize int) {
 		c := test.MustRunCluster(t, 1)
 		defer c.Close()
diff --git a/fragment.go b/fragment.go
index 9dc5b205b..c5991427c 100644
--- a/fragment.go
+++ b/fragment.go
@@ -3025,13 +3025,15 @@ func (f *fragment) rows(ctx context.Context, tx Tx, start uint64, filters ...row
 
 // unprotectedRows calls rows without grabbing the mutex.
 func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, filters ...rowFilter) ([]uint64, error) {
+	rows := make([]uint64, 0)
 	startKey := rowToKey(start)
 	i, _, err := tx.ContainerIterator(f.index, f.field, f.view, f.shard, startKey)
 	if err != nil {
 		return nil, err
+	} else if i == nil {
+		return rows, nil
 	}
 	defer i.Close() // must close iterators allocated on a Tx
-	rows := make([]uint64, 0)
 	var lastRow uint64 = math.MaxUint64
 
 	// Loop over the existing containers.
diff --git a/fragment_internal_test.go b/fragment_internal_test.go
index ff0de833f..c7de1a632 100644
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@@ -1786,6 +1786,8 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
 
 // Ensure a fragment's cache can be persisted between restarts.
 func TestFragment_RankCache_Persistence(t *testing.T) {
+	skipForRBF(t)
+
 	index := mustOpenIndex(IndexOptions{})
 	defer index.Close()
 
@@ -1847,6 +1849,8 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
 
 // Ensure a fragment can be copied to another fragment.
 func TestFragment_WriteTo_ReadFrom(t *testing.T) {
+	skipForRBF(t)
+
 	f0, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
 	_ = idx
 	defer f0.Clean(t)
@@ -4196,6 +4200,8 @@ func TestFragmentRowIterator(t *testing.T) {
 	})
 
 	t.Run("skipped rows wrapped", func(t *testing.T) {
+		skipForRBF(t)
+
 		f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked)
 		_ = idx
 		defer f.Clean(t)
@@ -4396,6 +4402,8 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
 			row, id, _, wrapped, err := iter.Next()
 			if err != nil {
 				t.Fatal(err)
+			} else if row == nil {
+				t.Fatal("expected row")
 			}
 			if id != i%8 {
 				t.Errorf("expected row %d but got %d", i%8, id)
@@ -5306,6 +5314,7 @@ func check(t *testing.T, tx Tx, f *fragment, exp map[uint64]map[uint64]struct{})
 }
 
 func TestImportValueConcurrent(t *testing.T) {
+	skipForRBF(t)
 
 	f, idx := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0)
 	switch idx.Txf.TxType() {
@@ -5631,3 +5640,9 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) {
 		t.Fatalf("expected nothing got %v", res)
 	}
 }
+
+func skipForRBF(tb testing.TB) {
+	if os.Getenv("PILOSA_TXSRC") == "rbf" {
+		tb.Skip("skip for RBF")
+	}
+}
diff --git a/holder_test.go b/holder_test.go
index 4280dadbd..2dfc55dc7 100644
--- a/holder_test.go
+++ b/holder_test.go
@@ -33,6 +33,7 @@ import (
 
 func TestHolder_Open(t *testing.T) {
 	skipForBadger := os.Getenv("PILOSA_TXSRC") == "badger"
+	skipForRBF := os.Getenv("PILOSA_TXSRC") == "rbf"
 
 	t.Run("ErrIndexName", func(t *testing.T) {
 		h := test.MustOpenHolder()
@@ -169,6 +170,8 @@ func TestHolder_Open(t *testing.T) {
 	t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
 		if skipForBadger {
 			t.Skip("skipping for badger")
+		} else if skipForRBF {
+			t.Skip("skipping for rbf")
 		}
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
@@ -208,6 +211,8 @@ func TestHolder_Open(t *testing.T) {
 	t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
 		if skipForBadger {
 			t.Skip("skipping for badger")
+		} else if skipForRBF {
+			t.Skip("skipping for rbf")
 		}
 
 		h := test.MustOpenHolder()
@@ -244,6 +249,8 @@ func TestHolder_Open(t *testing.T) {
 	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
 		if skipForBadger {
 			t.Skip("skipping for badger")
+		} else if skipForRBF {
+			t.Skip("skipping for rbf")
 		}
 
 		h := test.MustOpenHolder()
@@ -410,6 +417,8 @@ func TestHolder_HasData(t *testing.T) {
 
 // Ensure holder can delete an index and its underlying files.
 func TestHolder_DeleteIndex(t *testing.T) {
+	skipForRBF(t)
+
 	hldr := test.MustOpenHolder()
 	defer hldr.Close()
 
@@ -705,6 +714,8 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 
 // Ensure holder can sync integer views with a remote holder.
 func TestHolderSyncer_IntField(t *testing.T) {
+	skipForRBF(t)
+
 	t.Run("BasicSync", func(t *testing.T) {
 		c := test.MustNewCluster(t, 2)
 		c[0].Config.Cluster.ReplicaN = 2
diff --git a/mmap_test.go b/mmap_test.go
index 43321cea7..ab9643c73 100644
--- a/mmap_test.go
+++ b/mmap_test.go
@@ -86,6 +86,8 @@ func forceSnapshotsCheckMapping(t *testing.T) {
 // in newGeneration in generation.go. So this is probably useless but it's
 // a failure mode we've been bitten by once...
 func TestMmapBehavior(t *testing.T) {
+	skipForRBF(t)
+
 	var changed bool
 	var original uint64
 	defer func() {
diff --git a/pilosa_test.go b/pilosa_test.go
index 59b6fe082..55a5bbd65 100644
--- a/pilosa_test.go
+++ b/pilosa_test.go
@@ -15,6 +15,7 @@
 package pilosa_test
 
 import (
+	"os"
 	"strings"
 	"testing"
 
@@ -55,3 +56,9 @@ func TestAddressWithDefaults(t *testing.T) {
 		}
 	}
 }
+
+func skipForRBF(tb testing.TB) {
+	if os.Getenv("PILOSA_TXSRC") == "rbf" {
+		tb.Skip("skip for RBF")
+	}
+}
diff --git a/rbf/cursor.go b/rbf/cursor.go
index 9bb59fb6b..ed0a9f849 100644
--- a/rbf/cursor.go
+++ b/rbf/cursor.go
@@ -84,7 +84,7 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) {
 	}
 	return runs, true
 }
-func checkRun(runs []roaring.Interval16, key uint64) leafCell {
+func checkRun(runs []roaring.Interval16, bitN int, key uint64) leafCell {
 	if len(runs) >= RLEMaxSize {
 		//convertToBitmap
 		bitmap := make([]uint64, BitmapN)
@@ -123,9 +123,9 @@ func checkRun(runs []roaring.Interval16, key uint64) leafCell {
 			n += popcount(v)
 		}
 
-		return leafCell{Key: key, N: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)}
+		return leafCell{Key: key, N: int(n), BitN: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)}
 	}
-	return leafCell{Key: key, N: len(runs), Type: ContainerTypeRLE, Data: fromInterval16(runs)}
+	return leafCell{Key: key, N: len(runs), BitN: int(bitN + 1), Type: ContainerTypeRLE, Data: fromInterval16(runs)}
 }
 
 // Add sets a bit on the underlying bitmap.
@@ -136,7 +136,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
 	if exact, err := c.Seek(hi); err != nil {
 		return false, err
 	} else if !exact {
-		return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, Data: fromArray16([]uint16{lo})})
+		return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, BitN: 1, Data: fromArray16([]uint16{lo})})
 	}
 
 	// If the container exists and bit is not set then update the page.
@@ -155,7 +155,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
 		copy(other, a[:i])
 		other[i] = lo
 		copy(other[i+1:], a[i:])
-		return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)})
+		return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN + 1, Data: fromArray16(other)})
 
 	case ContainerTypeRLE:
 		runs := toInterval16(cell.Data)
@@ -163,7 +163,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
 		copy(c.rle[:], runs)
 		run, added := runAdd(c.rle[:len(runs)], lo)
 		if added {
-			leaf := checkRun(run, cell.Key)
+			leaf := checkRun(run, cell.BitN, cell.Key)
 			return true, c.putLeafCell(leaf)
 		}
 		return false, nil
@@ -185,6 +185,8 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
 		if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil {
 			return false, err
 		}
+		// TODO(bbj): Update parent cell with new BitN.
+
 		return true, nil
 	default:
 		return false, fmt.Errorf("rbf.Cursor.Add(): invalid container type: %d", cell.Type)
@@ -220,7 +222,7 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
 		other := make([]uint16, len(a)-1)
 		copy(other[:i], a[:i])
 		copy(other[i:], a[i+1:])
-		return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)})
+		return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN - 1, Data: fromArray16(other)})
 
 	case ContainerTypeRLE:
 		r := toInterval16(cell.Data)
@@ -265,6 +267,8 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
 		if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil {
 			return false, err
 		}
+
+		// TODO(bbj): Update parent cell to decrement BitN.
 		return true, nil
 	default:
 		return false, fmt.Errorf("rbf.Cursor.Add(): invalid container type: %d", cell.Type)
@@ -356,7 +360,10 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
 		}
 		in.Data = fromArray64(a)
 		cell.Type = ContainerTypeBitmapPtr
-		bitmapPgno, _ := c.tx.allocate()
+		bitmapPgno, err := c.tx.allocate()
+		if err != nil {
+			return err
+		}
 		cell.Data = fromPgno(bitmapPgno)
 	}
 
@@ -812,6 +819,7 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) {
 		if err != nil {
 			return false, err
 		}
+
 		switch typ := readFlags(buf); typ {
 		case PageTypeBranch:
 			n := readCellN(buf)
@@ -1089,6 +1097,7 @@ func (c *Cursor) goNextPage() error {
 func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) {
 	result.Key = key
 	result.N = int(c.N())
+	result.BitN = int(c.N())
 	result.Type = ContainerTypeNone
 	if c.N() == 0 {
 		return
@@ -1136,7 +1145,7 @@ func (c *Cursor) merge(key uint64, data *roaring.Container) (bool, error) {
 		if err != nil {
 			return false, errors.Wrap(err, "cursor.merge")
 		}
-		container = roaring.NewContainerBitmap(cell.N, d)
+		container = roaring.NewContainerBitmap(cell.BitN, d)
 	case ContainerTypeRLE:
 		d := toInterval16(cell.Data)
 		container = roaring.NewContainerRun(d)
diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go
index 0db356177..bd8e67203 100644
--- a/rbf/cursor_test.go
+++ b/rbf/cursor_test.go
@@ -31,7 +31,7 @@ func TestCursor_FirstNext(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
@@ -95,7 +95,7 @@ func TestCursor_FirstNext_Quick(t *testing.T) {
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
 		tx := MustBegin(t, db, true)
-		defer MustRollback(t, tx)
+		defer tx.Rollback()
 
 		// Insert values in random order.
 		if err := tx.CreateBitmap("x"); err != nil {
@@ -153,7 +153,7 @@ func TestCursor_LastPrev(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
@@ -217,7 +217,7 @@ func TestCursor_LastPrev_Quick(t *testing.T) {
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
 		tx := MustBegin(t, db, true)
-		defer MustRollback(t, tx)
+		defer tx.Rollback()
 
 		// Insert values in random order.
 		if err := tx.CreateBitmap("x"); err != nil {
@@ -276,7 +276,7 @@ func TestCursor_Union(t *testing.T) {
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
 		tx := MustBegin(t, db, true)
-		defer MustRollback(t, tx)
+		defer tx.Rollback()
 
 		if err := tx.CreateBitmap("x"); err != nil {
 			t.Fatal(err)
@@ -322,7 +322,7 @@ func TestCursor_Union(t *testing.T) {
 			db := MustOpenDB(t)
 			defer MustCloseDB(t, db)
 			tx := MustBegin(t, db, true)
-			defer MustRollback(t, tx)
+			defer tx.Rollback()
 			values := GenerateValues(rand, 10000)
 			rows := ToRows(values)
 
@@ -356,7 +356,7 @@ func TestCursor_Intersect(t *testing.T) {
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
 		tx := MustBegin(t, db, true)
-		defer MustRollback(t, tx)
+		defer tx.Rollback()
 
 		row := make([]uint64, rbf.ShardWidth/64)
 
@@ -403,7 +403,7 @@ func TestCursor_Intersect(t *testing.T) {
 			db := MustOpenDB(t)
 			defer MustCloseDB(t, db)
 			tx := MustBegin(t, db, true)
-			defer MustRollback(t, tx)
+			defer tx.Rollback()
 			values := GenerateValues(rand, rand.Intn(10000))
 			rows := ToRows(values)
 
@@ -447,7 +447,7 @@ func TestCursor_AddRoaring(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
@@ -466,7 +466,7 @@ func TestCursor_AddRoaring(t *testing.T) {
 			return bm
 		}(),
 		wantChanged: false,
-		wantErr:     true},
+		wantErr:     false},
 		{
 			name:      "initial Array",
 			fieldview: "x",
@@ -614,7 +614,7 @@ func TestCursor_RLETesting(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	//setup RLE
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
@@ -696,10 +696,10 @@ func TestCursor_RLETesting(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			changed, err := tx.Add("x", tt.args...)
+			changeCount, err := tx.Add("x", tt.args...)
 			if tt.wantErr && err == nil {
 				t.Errorf("No Error %v", err)
-			} else if tt.wantChanged && !changed {
+			} else if tt.wantChanged && changeCount == 0 {
 				t.Errorf("No Change %v", err)
 			} else if err != nil {
 				t.Fatal(err)
@@ -734,7 +734,7 @@ func TestCursor_RLEConversion(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	//setup RLE with full container
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
@@ -840,7 +840,7 @@ func TestCursor_UpdateBranchCells(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
 	}
@@ -914,7 +914,7 @@ func TestCursor_SplitBranchCells(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
 	}
@@ -964,7 +964,7 @@ func TestCursor_RemoveCells(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
 	}
@@ -1006,7 +1006,7 @@ func TestCursor_PlayContainer(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
 	}
@@ -1041,7 +1041,7 @@ func TestCursor_OneBitmap(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
 	}
@@ -1075,7 +1075,7 @@ func TestCursor_GenerateAll(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
 	}
diff --git a/rbf/cursorx.go b/rbf/cursorx.go
index 8a317c184..54febc5b0 100644
--- a/rbf/cursorx.go
+++ b/rbf/cursorx.go
@@ -93,10 +93,11 @@ func (c *Cursor) Dump(name string) {
 	fmt.Fprintf(bufStdout, "\n}")
 	bufStdout.Flush()
 }
-func (c *Cursor) Row(rowID uint64) (*roaring.Bitmap, error) {
+
+func (c *Cursor) Row(shard, rowID uint64) (*roaring.Bitmap, error) {
 	base := rowID * ShardWidth
 
-	offset := uint64(c.tx.db.Shard * ShardWidth)
+	offset := uint64(shard * ShardWidth)
 	off := highbits(offset)
 	hi0, hi1 := highbits(base), highbits((rowID+1)*ShardWidth)
 	c.stack.index = 0
diff --git a/rbf/db.go b/rbf/db.go
index 3bb960f71..9b642c1fb 100644
--- a/rbf/db.go
+++ b/rbf/db.go
@@ -54,27 +54,18 @@ type DB struct {
 
 	// The maximum allowed database size. Required by mmap.
 	MaxSize int64
-	Shard   int
 }
 
 // NewDB returns a new instance of DB.
 func NewDB(path string) *DB {
-	return NewDBWithShard(path, 0)
-}
-func NewDBWithShard(path string, shard int) *DB {
 	return &DB{
 		txs:     make(map[*Tx]struct{}),
 		pageMap: immutable.NewMap(&uint32Hasher{}),
 		Path:    path,
 		MaxSize: DefaultMaxSize,
-		Shard:   shard,
 	}
 }
 
-func (db *DB) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
-	panic("TODO: implement rbf.DB.DeleteFragment")
-}
-
 // DataPath returns the path to the data file for the DB.
 func (db *DB) DataPath() string {
 	return filepath.Join(db.Path, "data")
@@ -101,7 +92,7 @@ func (db *DB) Open() (err error) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
 
-	if err := os.MkdirAll(filepath.Dir(db.Path), 0755); err != nil {
+	if err := os.MkdirAll(db.Path, 0755); err != nil {
 		return err
 	} else if db.file, err = os.OpenFile(db.DataPath(), os.O_WRONLY|os.O_CREATE, 0666); err != nil {
 		return fmt.Errorf("open file: %w", err)
@@ -573,7 +564,7 @@ func (db *DB) Begin(writable bool) (_ *Tx, err error) {
 	// This page is only written at the end of a dirty transaction.
 	page, err := db.readPage(db.pageMap, 0)
 	if err != nil {
-		_ = tx.Rollback()
+		tx.Rollback()
 		return nil, err
 	}
 	copy(tx.meta[:], page)
@@ -617,7 +608,7 @@ func (db *DB) Check() error {
 	if err != nil {
 		return err
 	}
-	defer func() { _ = tx.Rollback() }()
+	defer tx.Rollback()
 	return tx.Check()
 }
 
diff --git a/rbf/db_test.go b/rbf/db_test.go
index f68b0f26e..ef4e75ade 100644
--- a/rbf/db_test.go
+++ b/rbf/db_test.go
@@ -119,14 +119,13 @@ func TestDB_Recovery(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		defer MustRollback(t, tx)
+		defer tx.Rollback()
 
 		if exists, err := tx.Contains("x", uint64(len(a))); exists || err != nil {
 			t.Fatalf("Contains()=<%v,%#v>", exists, err)
 		} else if exists, err := tx.Contains("x", uint64(len(a)-1)); !exists || err != nil {
 			t.Fatalf("Contains()=<%v,%#v>", exists, err)
-		} else if err := tx.Rollback(); err != nil {
-			t.Fatal(err)
 		}
+		tx.Rollback()
 	})
 }
diff --git a/rbf/rbf.go b/rbf/rbf.go
index 443b2f940..2aa3a473a 100644
--- a/rbf/rbf.go
+++ b/rbf/rbf.go
@@ -21,8 +21,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
+	"os"
 	"unsafe"
 
+	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/shardwidth"
 )
 
@@ -81,6 +84,8 @@ var (
 	ErrTxClosed           = errors.New("transaction closed")
 	ErrTxNotWritable      = errors.New("transaction not writable")
 	ErrBitmapNameRequired = errors.New("bitmap name required")
+	ErrBitmapNotFound     = errors.New("bitmap not found")
+	ErrBitmapExists       = errors.New("bitmap already exists")
 )
 
 // Debug is just a temporary flag used for debugging.
@@ -259,6 +264,7 @@ type leafCell struct {
 	Key  uint64
 	Type int
 	N    int
+	BitN int
 	Data []byte
 }
 
@@ -361,6 +367,49 @@ func (c *leafCell) firstValue() uint16 {
 	}
 }
 
+// lastValue the last value from the container.
+func (c *leafCell) lastValue() uint16 {
+	switch c.Type {
+	case ContainerTypeArray:
+		a := toArray16(c.Data)
+		return a[len(a)-1]
+	case ContainerTypeRLE:
+		r := toInterval16(c.Data)
+		return r[len(r)-1].Last
+	case ContainerTypeBitmap:
+		a := toArray64(c.Data)
+		for i := len(a) - 1; i >= 0; i-- {
+			for j := 63; j >= 0; j-- {
+				if a[i]&(1<<j) != 0 {
+					return (uint16(i) * 64) + uint16(j)
+				}
+			}
+		}
+		panic(fmt.Sprintf("rbf.leafCell.firstValue(): no values set in bitmap container: key=%d", c.Key))
+	default:
+		panic(fmt.Sprintf("invalid container type: %d", c.Type))
+	}
+}
+
+// countRange returns the bit count within the given range.
+func (c *leafCell) countRange(start, end uint16) (n int) {
+	// If the full range is being queried, simply use the precalculated count.
+	if start == 0 && end == math.MaxUint16 {
+		return c.BitN
+	}
+
+	switch c.Type {
+	case ContainerTypeArray:
+		return int(roaring.ArrayCountRange(toArray16(c.Data), int32(start), int32(end)))
+	case ContainerTypeRLE:
+		return int(roaring.RunCountRange(toInterval16(c.Data), int32(start), int32(end)))
+	case ContainerTypeBitmap:
+		return int(roaring.BitmapCountRange(toArray64(c.Data), int32(start), int32(end)))
+	default:
+		panic(fmt.Sprintf("invalid container type: %d", c.Type))
+	}
+}
+
 func readLeafCellKey(page []byte, i int) uint64 {
 	offset := readCellOffset(page, i)
 	return *(*uint64)(unsafe.Pointer(&page[offset]))
@@ -373,7 +422,8 @@ func readLeafCell(page []byte, i int) leafCell {
 	var cell leafCell
 	cell.Key = *(*uint64)(unsafe.Pointer(&buf[0]))
 	cell.Type = int(*(*uint32)(unsafe.Pointer(&buf[8])))
-	cell.N = int(*(*uint32)(unsafe.Pointer(&buf[12])))
+	cell.N = int(*(*uint16)(unsafe.Pointer(&buf[12])))
+	cell.BitN = int(*(*uint16)(unsafe.Pointer(&buf[14])))
 
 	switch cell.Type {
 	case ContainerTypeArray:
@@ -410,7 +460,8 @@ func writeLeafCell(page []byte, i, offset int, cell leafCell) {
 	writeCellOffset(page, i, offset)
 	*(*uint64)(unsafe.Pointer(&page[offset])) = cell.Key
 	*(*uint32)(unsafe.Pointer(&page[offset+8])) = uint32(cell.Type)
-	*(*uint32)(unsafe.Pointer(&page[offset+12])) = uint32(cell.N)
+	*(*uint16)(unsafe.Pointer(&page[offset+12])) = uint16(cell.N)
+	*(*uint16)(unsafe.Pointer(&page[offset+14])) = uint16(cell.BitN)
 	assert(offset+16+len(cell.Data) <= PageSize)
 	copy(page[offset+16:], cell.Data)
 }
@@ -486,8 +537,11 @@ func search(n int, f func(int) int) (index int, exact bool) {
 	return i, false
 }
 
-/*
-func pagedumpi(b []byte, indent string, writer io.Writer) {
+func Pagedump(b []byte, indent string, writer io.Writer) {
+	if writer == nil {
+		writer = os.Stderr
+	}
+
 	pgno := readPageNo(b)
 	if pgno == Magic32() {
 		fmt.Fprintf(writer, "==META\n")
@@ -501,6 +555,7 @@ func pagedumpi(b []byte, indent string, writer io.Writer) {
 	// the page alone so this will output !PAGE for bitmap pages & invalid pages.
 	switch {
 	case flags&PageTypeLeaf != 0:
+		fmt.Fprintf(writer, "==LEAF pgno=%d flags=%d n=%d\n", pgno, flags, cellN)
 		for i := 0; i < cellN; i++ {
 			cell := readLeafCell(b, i)
 			switch cell.Type {
@@ -525,7 +580,6 @@ func pagedumpi(b []byte, indent string, writer io.Writer) {
 		fmt.Fprintf(writer, "==!PAGE %d flags=%d\n", pgno, flags)
 	}
 }
-*/
 
 func Walk(tx *Tx, pgno uint32, v func(uint32, []*RootRecord)) {
 	for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
@@ -563,3 +617,8 @@ func RowValues(b []uint64) []uint64 {
 	}
 	return a
 }
+
+// func caller(skip int) string {
+// 	_, file, line, _ := runtime.Caller(skip + 1)
+// 	return fmt.Sprintf("%s:%d", file, line)
+// }
diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go
index a576dbd5e..8b4768cd5 100644
--- a/rbf/rbf_test.go
+++ b/rbf/rbf_test.go
@@ -119,14 +119,6 @@ func MustBegin(tb testing.TB, db *rbf.DB, writable bool) *rbf.Tx {
 	return tx
 }
 
-// MustRollback rolls back a transaction or fails.
-func MustRollback(tb testing.TB, tx *rbf.Tx) {
-	tb.Helper()
-	if err := tx.Rollback(); err != nil && err != rbf.ErrTxClosed {
-		tb.Logf("rollback error: %q", err)
-	}
-}
-
 // MustAddRandom adds values to a bitmap in a random order.
 func MustAddRandom(tb testing.TB, rand *rand.Rand, tx *rbf.Tx, name string, values ...uint64) {
 	tb.Helper()
diff --git a/rbf/tx.go b/rbf/tx.go
index 8740697dc..83f1ae244 100644
--- a/rbf/tx.go
+++ b/rbf/tx.go
@@ -16,7 +16,10 @@ package rbf
 import (
 	"fmt"
 	"io"
+	"math"
 	"sort"
+	"strings"
+	"sync"
 
 	"github.com/benbjohnson/immutable"
 	"github.com/pilosa/pilosa/v2/roaring"
@@ -24,6 +27,7 @@ import (
 
 // Tx represents a transaction.
 type Tx struct {
+	mu       sync.RWMutex
 	db       *DB            // parent db
 	meta     [PageSize]byte // copy of current meta page
 	walID    int64          // max WAL ID at start of tx
@@ -32,8 +36,16 @@ type Tx struct {
 	dirty    bool           // if true, changes have been made
 }
 
+// Writable returns true if the transaction can mutate data.
+func (tx *Tx) Writable() bool {
+	return tx.writable
+}
+
 // Commit completes the transaction and persists data changes.
 func (tx *Tx) Commit() error {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+
 	if tx.db == nil {
 		return ErrTxClosed
 	}
@@ -57,9 +69,14 @@ func (tx *Tx) Commit() error {
 	return tx.db.removeTx(tx)
 }
 
-func (tx *Tx) Rollback() error {
+func (tx *Tx) Rollback() {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+
+	// TODO(bbj): Invalidate DB if rollback fails. Possibly attempt reopen?
+
 	if tx.db == nil {
-		return ErrTxClosed
+		return
 	}
 
 	// If any pages have been written, ensure we write a new meta page with
@@ -67,29 +84,26 @@ func (tx *Tx) Rollback() error {
 	// discard pages in the transaction during playback of the WAL on open.
 	if tx.dirty {
 		if err := tx.writeMetaPage(MetaPageFlagRollback); err != nil {
-			return err
+			panic(err)
 		} else if err := tx.db.SyncWAL(); err != nil {
-			return err
+			panic(err)
 		}
 	}
 
-	if err := tx.db.checkpoint(); err != nil {
-		return err
-	}
+	_ = tx.db.checkpoint() // TODO: Check error
 
 	// Disconnect transaction from DB.
-	err := tx.db.removeTx(tx)
-	_ = err
-	/*
-		if err != nil {
-			//TODO need to fix this error
-		}
-	*/
-	return nil
+	_ = tx.db.removeTx(tx) // TODO: Check error
 }
 
 // Root returns the root page number for a bitmap. Returns 0 if the bitmap does not exist.
 func (tx *Tx) Root(name string) (uint32, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+	return tx.root(name)
+}
+
+func (tx *Tx) root(name string) (uint32, error) {
 	records, err := tx.rootRecords()
 	if err != nil {
 		return 0, err
@@ -97,14 +111,43 @@ func (tx *Tx) Root(name string) (uint32, error) {
 
 	i := sort.Search(len(records), func(i int) bool { return records[i].Name >= name })
 	if i >= len(records) || records[i].Name != name {
-		return 0, fmt.Errorf("bitmap not found: %q", name)
+		return 0, ErrBitmapNotFound
 	}
 	return records[i].Pgno, nil
 }
 
+// BitmapNames returns a list of all bitmap names.
+func (tx *Tx) BitmapNames() ([]string, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	if tx.db == nil {
+		return nil, ErrTxClosed
+	}
+
+	// Read list of root records.
+	records, err := tx.rootRecords()
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert to a list of strings.
+	names := make([]string, len(records))
+	for i := range records {
+		names[i] = records[i].Name
+	}
+	return names, nil
+}
+
 // CreateBitmap creates a new empty bitmap with the given name.
 // Returns an error if the bitmap already exists.
 func (tx *Tx) CreateBitmap(name string) error {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+	return tx.createBitmap(name)
+}
+
+func (tx *Tx) createBitmap(name string) error {
 	if tx.db == nil {
 		return ErrTxClosed
 	} else if !tx.writable {
@@ -122,7 +165,7 @@ func (tx *Tx) CreateBitmap(name string) error {
 	// Find btree by name. Exit if already exists.
 	index := sort.Search(len(records), func(i int) bool { return records[i].Name >= name })
 	if index < len(records) && records[index].Name == name {
-		return fmt.Errorf("bitmap already exists: %q", name)
+		return ErrBitmapExists
 	}
 	//fmt.Println("CREATE BITMAP", name, index)
 
@@ -153,6 +196,22 @@ func (tx *Tx) CreateBitmap(name string) error {
 	return nil
 }
 
+// CreateBitmapIfNotExists creates a new empty bitmap with the given name.
+// This is a no-op if the bitmap already exists.
+func (tx *Tx) CreateBitmapIfNotExists(name string) error {
+	if err := tx.CreateBitmap(name); err != nil && err != ErrBitmapExists {
+		return err
+	}
+	return nil
+}
+
+func (tx *Tx) createBitmapIfNotExists(name string) error {
+	if err := tx.createBitmap(name); err != nil && err != ErrBitmapExists {
+		return err
+	}
+	return nil
+}
+
 /*
 func dump(r []*RootRecord) {
 	for _, i := range r {
@@ -165,6 +224,9 @@ func dump(r []*RootRecord) {
 // DeleteBitmap removes a bitmap with the given name.
 // Returns an error if the bitmap does not exist.
 func (tx *Tx) DeleteBitmap(name string) error {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+
 	if tx.db == nil {
 		return ErrTxClosed
 	} else if !tx.writable {
@@ -200,9 +262,55 @@ func (tx *Tx) DeleteBitmap(name string) error {
 	return nil
 }
 
+// DeleteBitmapsWithPrefix removes all bitmaps with a given prefix.
+func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+
+	if tx.db == nil {
+		return ErrTxClosed
+	} else if !tx.writable {
+		return ErrTxNotWritable
+	}
+
+	// Read list of root records.
+	records, err := tx.rootRecords()
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < len(records); i++ {
+		record := records[i]
+
+		// Skip bitmaps without matching prefix.
+		if !strings.HasPrefix(record.Name, prefix) {
+			continue
+		}
+
+		// Deallocate all pages in the tree.
+		if err := tx.deallocateTree(record.Pgno); err != nil {
+			return err
+		}
+
+		// Delete from record list.
+		records = append(records[:i], records[i+1:]...)
+		i--
+	}
+
+	// Rewrite record pages.
+	if err := tx.writeRootRecordPages(records); err != nil {
+		return fmt.Errorf("write bitmaps: %w", err)
+	}
+
+	return nil
+}
+
 // RenameBitmap updates the name of an existing bitmap.
 // Returns an error if the bitmap does not exist.
 func (tx *Tx) RenameBitmap(oldname, newname string) error {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+
 	if tx.db == nil {
 		return ErrTxClosed
 	} else if !tx.writable {
@@ -314,78 +422,103 @@ func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) {
 }
 
 // Add sets a given bit on the bitmap.
-func (tx *Tx) Add(name string, a ...uint64) (changed bool, err error) {
+func (tx *Tx) Add(name string, a ...uint64) (changeCount int, err error) {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+
 	if tx.db == nil {
-		return false, ErrTxClosed
+		return 0, ErrTxClosed
 	} else if !tx.writable {
-		return false, ErrTxNotWritable
+		return 0, ErrTxNotWritable
 	} else if name == "" {
-		return false, ErrBitmapNameRequired
+		return 0, ErrBitmapNameRequired
 	}
 
-	c, err := tx.Cursor(name)
+	if err := tx.createBitmapIfNotExists(name); err != nil {
+		return 0, err
+	}
+
+	c, err := tx.cursor(name)
 	if err != nil {
-		return false, err
+		return 0, err
 	}
 	for _, v := range a {
 		if vchanged, err := c.Add(v); err != nil {
-			return changed, err
+			return changeCount, err
 		} else if vchanged {
-			changed = true
+			changeCount++
 		}
 	}
-	return changed, nil
+	return changeCount, nil
 }
 
 // Remove unsets a given bit on the bitmap.
-func (tx *Tx) Remove(name string, a ...uint64) (changed bool, err error) {
+func (tx *Tx) Remove(name string, a ...uint64) (changeCount int, err error) {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+
 	if tx.db == nil {
-		return false, ErrTxClosed
+		return 0, ErrTxClosed
 	} else if !tx.writable {
-		return false, ErrTxNotWritable
+		return 0, ErrTxNotWritable
 	} else if name == "" {
-		return false, ErrBitmapNameRequired
+		return 0, ErrBitmapNameRequired
 	}
 
-	c, err := tx.Cursor(name)
+	c, err := tx.cursor(name)
 	if err != nil {
-		return false, err
+		return 0, err
+	} else if c == nil {
+		return 0, nil
 	}
 	for _, v := range a {
 		if vchanged, err := c.Remove(v); err != nil {
-			return changed, err
+			return changeCount, err
 		} else if vchanged {
-			changed = true
+			changeCount++
 		}
 	}
-	return changed, nil
+	return changeCount, nil
 }
 
 // Contains returns true if the given bit is set on the bitmap.
 func (tx *Tx) Contains(name string, v uint64) (bool, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
 	if tx.db == nil {
 		return false, ErrTxClosed
 	} else if name == "" {
 		return false, ErrBitmapNameRequired
 	}
 
-	c, err := tx.Cursor(name)
+	c, err := tx.cursor(name)
 	if err != nil {
 		return false, err
+	} else if c == nil {
+		return false, nil
 	}
 	return c.Contains(v)
 }
 
 // Cursor returns an instance of a cursor this bitmap.
 func (tx *Tx) Cursor(name string) (*Cursor, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+	return tx.cursor(name)
+}
+
+func (tx *Tx) cursor(name string) (*Cursor, error) {
 	if tx.db == nil {
 		return nil, ErrTxClosed
 	} else if name == "" {
 		return nil, ErrBitmapNameRequired
 	}
 
-	root, err := tx.Root(name)
-	if err != nil {
+	root, err := tx.root(name)
+	if err == ErrBitmapNotFound {
+		return nil, nil
+	} else if err != nil {
 		return nil, err
 	}
 
@@ -394,8 +527,109 @@ func (tx *Tx) Cursor(name string) (*Cursor, error) {
 	return &c, nil
 }
 
+// RoaringBitmap returns a bitmap as a Roaring bitmap.
+func (tx *Tx) RoaringBitmap(name string) (*roaring.Bitmap, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	if tx.db == nil {
+		return nil, ErrTxClosed
+	} else if name == "" {
+		return nil, ErrBitmapNameRequired
+	}
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		return nil, err
+	} else if c == nil {
+		return roaring.NewSliceBitmap(), nil
+	}
+
+	other := roaring.NewSliceBitmap()
+	if err := c.First(); err == io.EOF {
+		return other, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	for {
+		if err := c.Next(); err == io.EOF {
+			return other, nil
+		} else if err != nil {
+			return nil, err
+		}
+
+		cell := c.cell()
+		other.Containers.Put(cell.Key, toContainer(cell, tx))
+	}
+}
+
+// Container returns a Roaring container by key.
+func (tx *Tx) Container(name string, key uint64) (*roaring.Container, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	if tx.db == nil {
+		return nil, ErrTxClosed
+	} else if name == "" {
+		return nil, ErrBitmapNameRequired
+	}
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		return nil, err
+	} else if c == nil {
+		return nil, err
+	} else if exact, err := c.Seek(key); err != nil || !exact {
+		return nil, err
+	}
+	return toContainer(c.cell(), tx), nil
+}
+
+// PutContainer inserts a container into a bitmap. Overwrites if key already exists.
+func (tx *Tx) PutContainer(name string, key uint64, cont *roaring.Container) error {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+
+	cell := ConvertToLeafArgs(key, cont)
+	if cell.BitN == 0 {
+		return nil
+	}
+
+	if err := tx.createBitmapIfNotExists(name); err != nil {
+		return err
+	}
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		return err
+	} else if _, err := c.Seek(cell.Key); err != nil {
+		return err
+	}
+	return c.putLeafCell(cell)
+}
+
+// RemoveContainer removes a container from the bitmap by key.
+func (tx *Tx) RemoveContainer(name string, key uint64) error {
+	tx.mu.Lock()
+	defer tx.mu.Unlock()
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		return err
+	} else if c == nil {
+		return nil
+	} else if exact, err := c.Seek(key); err != nil || !exact {
+		return err
+	}
+	return c.deleteLeafCell(key)
+}
+
 // Check verifies the integrity of the database.
 func (tx *Tx) Check() error {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
 	if tx.db == nil {
 		return ErrTxClosed
 	}
@@ -677,12 +911,20 @@ func (tx *Tx) writeMetaPage(flag uint32) error {
 }
 
 func (tx *Tx) AddRoaring(name string, bm *roaring.Bitmap) (changed bool, err error) {
-	c, err := tx.Cursor(name)
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	if err := tx.createBitmapIfNotExists(name); err != nil {
+		return false, err
+	}
+
+	c, err := tx.cursor(name)
 	if err != nil {
 		return false, err
 	}
 	return c.AddRoaring(bm)
 }
+
 func (tx *Tx) leafCellBitmap(pgno uint32) (uint32, []uint64, error) {
 	page, err := tx.readPage(pgno)
 	if err != nil {
@@ -690,3 +932,231 @@ func (tx *Tx) leafCellBitmap(pgno uint32) (uint32, []uint64, error) {
 	}
 	return pgno, toArray64(page), err
 }
+
+func (tx *Tx) ContainerIterator(name string, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		// TODO(bbj): Don't return error if bitmap is simply not found?
+		return nil, false, err
+	} else if c == nil {
+		return nil, false, nil
+	} else if err := c.First(); err != nil {
+		return nil, false, err
+	}
+	return &containerIterator{cursor: c}, true, nil
+}
+
+func (tx *Tx) ForEach(name string, fn func(i uint64) error) error {
+	return tx.ForEachRange(name, 0, math.MaxUint64, fn)
+}
+
+func (tx *Tx) ForEachRange(name string, start, end uint64, fn func(uint64) error) error {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		return err
+	} else if c == nil {
+		return nil
+	} else if _, err := c.Seek(highbits(start)); err != nil {
+		return err
+	}
+
+	for {
+		if err := c.Next(); err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		switch cell := c.cell(); cell.Type {
+		case ContainerTypeArray:
+			for _, lo := range toArray16(cell.Data) {
+				v := cell.Key<<16 | uint64(lo)
+				if v < start {
+					continue
+				} else if v > end {
+					return nil
+				} else if err := fn(v); err != nil {
+					return err
+				}
+			}
+		case ContainerTypeRLE:
+			for _, r := range toInterval16(cell.Data) {
+				for lo := int(r.Start); lo <= int(r.Last); lo++ {
+					v := cell.Key<<16 | uint64(lo)
+					if v < start {
+						continue
+					} else if v > end {
+						return nil
+					} else if err := fn(v); err != nil {
+						return err
+					}
+				}
+			}
+		case ContainerTypeBitmap:
+			for i, bits := range toArray64(cell.Data) {
+				for j := uint(0); j < 64; j++ {
+					if bits&(1<<j) != 0 {
+						continue
+					}
+
+					v := cell.Key<<16 | (uint64(i) * 64) | uint64(j)
+					if v < start {
+						continue
+					} else if v > end {
+						return nil
+					} else if err := fn(v); err != nil {
+						return err
+					}
+				}
+			}
+		default:
+			panic(fmt.Sprintf("invalid container type: %d", cell.Type))
+		}
+	}
+}
+
+func (tx *Tx) Count(name string) (uint64, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		return 0, err
+	} else if c == nil {
+		return 0, nil
+	} else if err := c.First(); err != nil {
+		return 0, err
+	}
+
+	var n uint64
+	for {
+		if err := c.Next(); err == io.EOF {
+			break
+		} else if err != nil {
+			return 0, err
+		}
+
+		n += uint64(c.cell().BitN)
+	}
+	return n, nil
+}
+
+func (tx *Tx) Max(name string) (uint64, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		return 0, err
+	} else if c == nil {
+		return 0, nil
+	} else if err := c.Last(); err == io.EOF {
+		return 0, nil
+	} else if err != nil {
+		return 0, err
+	}
+
+	cell := c.cell()
+	return uint64((cell.Key << 16) | uint64(cell.lastValue())), nil
+}
+
+func (tx *Tx) Min(name string) (uint64, bool, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		return 0, false, err
+	} else if c == nil {
+		return 0, false, nil
+	} else if err := c.First(); err == io.EOF {
+		return 0, false, nil
+	} else if err != nil {
+		return 0, false, err
+	}
+
+	cell := c.cell()
+	return uint64((cell.Key << 16) | uint64(cell.firstValue())), true, nil
+}
+
+func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error {
+	panic("TODO")
+}
+
+func (tx *Tx) CountRange(name string, start, end uint64) (uint64, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	c, err := tx.cursor(name)
+	if err != nil {
+		return 0, err
+	} else if c == nil {
+		return 0, nil
+	}
+
+	if err := c.First(); err == io.EOF {
+		return 0, nil
+	} else if err != nil {
+		return 0, err
+	}
+
+	var n uint64
+	for {
+		if err := c.Next(); err == io.EOF {
+			break
+		} else if err != nil {
+			return 0, err
+		}
+
+		cell := c.cell()
+		if cell.Key > highbits(end) {
+			break
+		}
+
+		if cell.Key == highbits(start) {
+			n += uint64(cell.countRange(lowbits(start), math.MaxUint16))
+		} else if cell.Key == highbits(end) {
+			n += uint64(cell.countRange(0, lowbits(end)))
+		} else {
+			n += uint64(cell.BitN)
+		}
+	}
+	return n, nil
+}
+
+func (tx *Tx) OffsetRange(name string, offset, start, end uint64) (*roaring.Bitmap, error) {
+	tx.mu.RLock()
+	defer tx.mu.RUnlock()
+
+	b, err := tx.RoaringBitmap(name)
+	if err != nil {
+		return nil, err
+	}
+	return b.OffsetRange(offset, start, end), nil
+}
+
+// containerIterator wraps Cursor to implement roaring.ContainerIterator.
+type containerIterator struct {
+	cursor *Cursor
+}
+
+// Close is a no-op. It exists to implement the roaring.ContainerIterator interface.
+func (itr *containerIterator) Close() {}
+
+// Next moves the iterator to the next container.
+func (itr *containerIterator) Next() bool {
+	err := itr.cursor.Next()
+	return err != nil
+}
+
+// Value returns the current key & container.
+func (itr *containerIterator) Value() (uint64, *roaring.Container) {
+	cell := itr.cursor.cell()
+	return cell.Key, toContainer(cell, itr.cursor.tx)
+}
diff --git a/rbf/tx_test.go b/rbf/tx_test.go
index 2af3f4bfc..85195ad1d 100644
--- a/rbf/tx_test.go
+++ b/rbf/tx_test.go
@@ -29,13 +29,13 @@ func TestTx_CommitRollback(t *testing.T) {
 		defer MustCloseDB(t, db)
 
 		// Create bitmap in transaction but rollback.
-		if tx, err := db.Begin(true); err != nil {
+		tx, err := db.Begin(true)
+		if err != nil {
 			t.Fatal(err)
 		} else if err := tx.CreateBitmap("x"); err != nil {
 			t.Fatal(err)
-		} else if err := tx.Rollback(); err != nil {
-			t.Fatal(err)
 		}
+		tx.Rollback()
 
 		// Create bitmap in transaction again but commit.
 		if tx, err := db.Begin(true); err != nil {
@@ -49,8 +49,8 @@ func TestTx_CommitRollback(t *testing.T) {
 		// Create bitmap again but it should fail as it already exists.
 		if tx, err := db.Begin(true); err != nil {
 			t.Fatal(err)
-		} else if err := tx.CreateBitmap("x"); err == nil || err.Error() != `bitmap already exists: "x"` {
-			_ = tx.Rollback()
+		} else if err := tx.CreateBitmap("x"); err == nil || err != rbf.ErrBitmapExists {
+			tx.Rollback()
 			t.Fatal(err)
 		} else if err := tx.Commit(); err != nil {
 			t.Fatal(err)
@@ -62,13 +62,13 @@ func TestTx_CommitRollback(t *testing.T) {
 		defer func() { MustCloseDB(t, db) }()
 
 		// Create bitmap in transaction but rollback.
-		if tx, err := db.Begin(true); err != nil {
+		tx, err := db.Begin(true)
+		if err != nil {
 			t.Fatal(err)
 		} else if err := tx.CreateBitmap("x"); err != nil {
 			t.Fatal(err)
-		} else if err := tx.Rollback(); err != nil {
-			t.Fatal(err)
 		}
+		tx.Rollback()
 		db = MustReopenDB(t, db)
 
 		// Create bitmap in transaction again but commit.
@@ -84,8 +84,8 @@ func TestTx_CommitRollback(t *testing.T) {
 		// Create bitmap again but it should fail as it already exists.
 		if tx, err := db.Begin(true); err != nil {
 			t.Fatal(err)
-		} else if err := tx.CreateBitmap("x"); err == nil || err.Error() != `bitmap already exists: "x"` {
-			_ = tx.Rollback()
+		} else if err := tx.CreateBitmap("x"); err == nil || err != rbf.ErrBitmapExists {
+			tx.Rollback()
 			t.Fatal(err)
 		} else if err := tx.Commit(); err != nil {
 			t.Fatal(err)
@@ -102,7 +102,7 @@ func TestTx_CommitRollback(t *testing.T) {
 		tx0 := MustBegin(t, db, true)
 		go func() {
 			<-ch0
-			_ = tx0.Rollback()
+			tx0.Rollback()
 		}()
 
 		// Start separate write transaction in different goroutine.
@@ -134,7 +134,7 @@ func TestTx_Add(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
@@ -167,7 +167,7 @@ func TestTx_DeleteBitmap(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 
 	// Create bitmap & add value.
 	if err := tx.CreateBitmap("x"); err != nil {
@@ -192,7 +192,7 @@ func TestTx_RenameBitmap(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 
 	// Create bitmap & add value.
 	if err := tx.CreateBitmap("x"); err != nil {
@@ -226,7 +226,7 @@ func TestTx_Add_Quick(t *testing.T) {
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
 		tx := MustBegin(t, db, true)
-		defer MustRollback(t, tx)
+		defer tx.Rollback()
 		values := GenerateValues(rand, 10000)
 
 		if err := tx.CreateBitmap("x"); err != nil {
@@ -265,7 +265,7 @@ func TestTx_AddRemove_Quick(t *testing.T) {
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
 		tx := MustBegin(t, db, true)
-		defer MustRollback(t, tx)
+		defer tx.Rollback()
 		values := GenerateValues(rand, 10000)
 
 		if err := tx.CreateBitmap("x"); err != nil {
@@ -314,7 +314,7 @@ func TestTx_Multiple_CreateBitmap(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	values := GenerateValues(rand, 2)
 
 	if err := tx.CreateBitmap("x/1"); err != nil {
@@ -332,7 +332,7 @@ func TestTx_Multiple_CreateBitmap(t *testing.T) {
 	}
 
 	tx1 := MustBegin(t, db, true)
-	defer func() { _ = tx1.Rollback() }()
+	defer tx1.Rollback()
 
 	if err := tx1.CreateBitmap("x/2"); err != nil {
 		t.Fatal(err)
@@ -353,7 +353,7 @@ func TestTx_CursorCrashArray(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
 	}
@@ -379,7 +379,7 @@ func TestTx_CursorCrashBitmap(t *testing.T) {
 	db := MustOpenDB(t)
 	defer MustCloseDB(t, db)
 	tx := MustBegin(t, db, true)
-	defer MustRollback(t, tx)
+	defer tx.Rollback()
 	if err := tx.CreateBitmap("x"); err != nil {
 		t.Fatal(err)
 	}
@@ -418,7 +418,7 @@ func BenchmarkTx_Add(b *testing.B) {
 					db := MustOpenDB(b)
 					defer MustCloseDB(b, db)
 					tx := MustBegin(b, db, true)
-					defer MustRollback(b, tx)
+					defer tx.Rollback()
 
 					for _, v := range values {
 						if _, err := tx.Add("x", v); err != nil {
@@ -446,7 +446,7 @@ func BenchmarkTx_Contains(b *testing.B) {
 			db := MustOpenDB(b)
 			defer MustCloseDB(b, db)
 			tx := MustBegin(b, db, true)
-			defer MustRollback(b, tx)
+			defer tx.Rollback()
 
 			b.ResetTimer()
 			t := time.Now()
diff --git a/roaring/roaring.go b/roaring/roaring.go
index 7f755f995..41ff21a17 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -2860,20 +2860,19 @@ func (c *Container) countRange(start, end int32) (n int32) {
 		return 0
 	}
 	if c.isArray() {
-		return c.arrayCountRange(start, end)
+		return ArrayCountRange(c.array(), start, end)
 	} else if c.isRun() {
-		return c.runCountRange(start, end)
+		return RunCountRange(c.runs(), start, end)
 	}
-	return c.bitmapCountRange(start, end)
+	return BitmapCountRange(c.bitmap(), start, end)
 }
 
-func (c *Container) arrayCountRange(start, end int32) (n int32) {
+func ArrayCountRange(array []uint16, start, end int32) (n int32) {
 	if roaringParanoia {
 		if start > end {
 			panic(fmt.Sprintf("counting in range but %v > %v", start, end))
 		}
 	}
-	array := c.array()
 	i := int32(sort.Search(len(array), func(i int) bool { return int32(array[i]) >= start }))
 	for ; i < int32(len(array)); i++ {
 		v := int32(array[i])
@@ -2885,7 +2884,7 @@ func (c *Container) arrayCountRange(start, end int32) (n int32) {
 	return n
 }
 
-func (c *Container) bitmapCountRange(start, end int32) int32 {
+func BitmapCountRange(bitmap []uint64, start, end int32) int32 {
 	if roaringParanoia {
 		if start > end {
 			panic(fmt.Sprintf("counting in range but %v > %v", start, end))
@@ -2894,7 +2893,6 @@ func (c *Container) bitmapCountRange(start, end int32) int32 {
 	var n uint64
 	i, j := start/64, end/64
 	// Special case when start and end fall in the same word.
-	bitmap := c.bitmap()
 	if i == j {
 		offi, offj := uint(start%64), uint(64-end%64)
 		n += popcount((bitmap[i] >> offi) << (offj + offi))
@@ -2921,13 +2919,13 @@ func (c *Container) bitmapCountRange(start, end int32) int32 {
 	return int32(n)
 }
 
-func (c *Container) runCountRange(start, end int32) (n int32) {
+// RunCountRange returns the ranged bit count for RLE pairs.
+func RunCountRange(runs []Interval16, start, end int32) (n int32) {
 	if roaringParanoia {
 		if start > end {
 			panic(fmt.Sprintf("counting in range but %v > %v", start, end))
 		}
 	}
-	runs := c.runs()
 	for _, iv := range runs {
 		// iv is before range
 		if int32(iv.Last) < start {
@@ -3837,12 +3835,12 @@ func (c *Container) check() error {
 			a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(array), c.N()))
 		}
 	} else if c.isRun() {
-		n := c.runCountRange(0, MaxContainerVal+1)
+		n := RunCountRange(c.runs(), 0, MaxContainerVal+1)
 		if n != c.N() {
 			a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.N()))
 		}
 	} else if c.isBitmap() {
-		if n := c.bitmapCountRange(0, MaxContainerVal+1); n != c.N() {
+		if n := BitmapCountRange(c.bitmap(), 0, MaxContainerVal+1); n != c.N() {
 			a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.N()))
 		}
 	} else {
@@ -4052,7 +4050,7 @@ func intersectionCountRunRun(a, b *Container) (n int32) {
 func intersectionCountBitmapRun(a, b *Container) (n int32) {
 	statsHit("intersectionCount/BitmapRun")
 	for _, iv := range b.runs() {
-		n += a.bitmapCountRange(int32(iv.Start), int32(iv.Last)+1)
+		n += BitmapCountRange(a.bitmap(), int32(iv.Start), int32(iv.Last)+1)
 	}
 	return n
 }
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index a3ae31527..42b34860e 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -131,7 +131,7 @@ func TestContainerRunAdd2(t *testing.T) {
 
 func TestRunCountRange(t *testing.T) {
 	c := NewContainerRun(nil)
-	cnt := c.runCountRange(2, 9)
+	cnt := RunCountRange(c.runs(), 2, 9)
 	if cnt != 0 {
 		t.Fatalf("should get 0 from empty container, but got: %v", cnt)
 	}
@@ -139,7 +139,7 @@ func TestRunCountRange(t *testing.T) {
 	c.add(6)
 	c.add(7)
 
-	cnt = c.runCountRange(2, 9)
+	cnt = RunCountRange(c.runs(), 2, 9)
 	if cnt != 3 {
 		t.Fatalf("should get 3 from interval within range, but got: %v", cnt)
 	}
@@ -149,52 +149,52 @@ func TestRunCountRange(t *testing.T) {
 	c.add(10)
 	c.add(11)
 
-	cnt = c.runCountRange(4, 8)
+	cnt = RunCountRange(c.runs(), 4, 8)
 	if cnt != 3 {
 		t.Fatalf("should get 3 from range overlaps front of interval, but got: %v", cnt)
 	}
 
-	cnt = c.runCountRange(5, 8)
+	cnt = RunCountRange(c.runs(), 5, 8)
 	if cnt != 3 {
 		t.Fatalf("should get 3 from range within interval, but got: %v", cnt)
 	}
 
-	cnt = c.runCountRange(6, 8)
+	cnt = RunCountRange(c.runs(), 6, 8)
 	if cnt != 2 {
 		t.Fatalf("should get 2 from range within interval, but got: %v", cnt)
 	}
 
-	cnt = c.runCountRange(3, 9)
+	cnt = RunCountRange(c.runs(), 3, 9)
 	if cnt != 4 {
 		t.Fatalf("should get 4 from range overlaps front of interval, but got: %v", cnt)
 	}
 
-	cnt = c.runCountRange(9, 14)
+	cnt = RunCountRange(c.runs(), 9, 14)
 	if cnt != 3 {
 		t.Fatalf("should get 3 from range overlaps back of interval, but got: %v", cnt)
 	}
 
-	cnt = c.runCountRange(8, 10)
+	cnt = RunCountRange(c.runs(), 8, 10)
 	if cnt != 2 {
 		t.Fatalf("should get 2 from range within interval, but got: %v", cnt)
 	}
 
-	cnt = c.runCountRange(8, 11)
+	cnt = RunCountRange(c.runs(), 8, 11)
 	if cnt != 3 {
 		t.Fatalf("should get 3 from range within interval, but got: %v", cnt)
 	}
 
-	cnt = c.runCountRange(8, 12)
+	cnt = RunCountRange(c.runs(), 8, 12)
 	if cnt != 4 {
 		t.Fatalf("should get 4 from range overlaps back of interval, but got: %v", cnt)
 	}
 
-	cnt = c.runCountRange(5, 12)
+	cnt = RunCountRange(c.runs(), 5, 12)
 	if cnt != 7 {
 		t.Fatalf("should get 7 from interval within range, but got: %v", cnt)
 	}
 
-	cnt = c.runCountRange(5, 11)
+	cnt = RunCountRange(c.runs(), 5, 11)
 	if cnt != 6 {
 		t.Fatalf("should get 6 from interval equal to range, but got: %v", cnt)
 	}
@@ -203,7 +203,7 @@ func TestRunCountRange(t *testing.T) {
 	c.add(19)
 	c.add(18)
 
-	cnt = c.runCountRange(1, 22)
+	cnt = RunCountRange(c.runs(), 1, 22)
 	if cnt != 10 {
 		t.Fatalf("should get 10 from multiple ranges in interval, but got: %v", cnt)
 	}
@@ -211,7 +211,7 @@ func TestRunCountRange(t *testing.T) {
 	c.add(13)
 	c.add(14)
 
-	cnt = c.runCountRange(6, 18)
+	cnt = RunCountRange(c.runs(), 6, 18)
 	if cnt != 9 {
 		t.Fatalf("should get 9 from multiple ranges overlapping both sides, but got: %v", cnt)
 	}
@@ -263,7 +263,7 @@ func TestBitmapCountRange(t *testing.T) {
 
 	for i, test := range tests {
 		c.setBitmap(test.bitmap[:])
-		if ret := c.bitmapCountRange(test.start, test.end); ret != test.exp {
+		if ret := BitmapCountRange(c.bitmap(), test.start, test.end); ret != test.exp {
 			t.Fatalf("test #%v count of %v from %v to %v should be %v but got %v", i, test.bitmap, test.start, test.end, test.exp, ret)
 		}
 	}
diff --git a/tx.go b/tx.go
index 1dbea6daf..94fbbb383 100644
--- a/tx.go
+++ b/tx.go
@@ -21,8 +21,10 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 
+	"github.com/pilosa/pilosa/v2/rbf"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pkg/errors"
 )
@@ -872,3 +874,147 @@ func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64
 	r = file
 	return
 }
+
+type RBFTx struct {
+	tx *rbf.Tx
+}
+
+func (tx *RBFTx) Type() string {
+	return RBFTxn
+}
+
+func (tx *RBFTx) Rollback() {
+	tx.tx.Rollback()
+}
+
+func (tx *RBFTx) Commit() error {
+	return tx.tx.Commit()
+}
+
+func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
+	return tx.tx.RoaringBitmap(rbfName(field, view, shard))
+}
+
+func (tx *RBFTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
+	return tx.tx.Container(rbfName(field, view, shard), key)
+}
+
+func (tx *RBFTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
+	return tx.tx.PutContainer(rbfName(field, view, shard), key, c)
+}
+
+func (tx *RBFTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
+	return tx.tx.RemoveContainer(rbfName(field, view, shard), key)
+}
+
+func (tx *RBFTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
+	return tx.tx.Add(rbfName(field, view, shard), a...)
+}
+
+func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+	return tx.tx.Remove(rbfName(field, view, shard), a...)
+}
+
+func (tx *RBFTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
+	return tx.tx.Contains(rbfName(field, view, shard), v)
+}
+
+func (tx *RBFTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
+	return tx.tx.ContainerIterator(rbfName(field, view, shard), key)
+}
+
+func (tx *RBFTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
+	return tx.tx.ForEach(rbfName(field, view, shard), fn)
+}
+
+func (tx *RBFTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
+	return tx.tx.ForEachRange(rbfName(field, view, shard), start, end, fn)
+}
+
+func (tx *RBFTx) Count(index, field, view string, shard uint64) (uint64, error) {
+	return tx.tx.Count(rbfName(field, view, shard))
+}
+
+func (tx *RBFTx) Max(index, field, view string, shard uint64) (uint64, error) {
+	return tx.tx.Max(rbfName(field, view, shard))
+}
+
+func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
+	return tx.tx.Min(rbfName(field, view, shard))
+}
+
+func (tx *RBFTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
+	return tx.tx.UnionInPlace(rbfName(field, view, shard), others...)
+}
+
+func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
+	return tx.tx.CountRange(rbfName(field, view, shard), start, end)
+}
+
+func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
+	return tx.tx.OffsetRange(rbfName(field, view, shard), offset, start, end)
+}
+
+func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {}
+
+func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
+	// TODO: Implement RBFTX.ImportRoaringBits"
+	return 0, make(map[uint64]int), nil
+}
+
+func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
+	panic("TODO: Implement RBFTx.RoaringBitmapReader()")
+}
+
+func (tx *RBFTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
+	prefix := rbfFieldViewPrefix(field, view)
+
+	names, err := tx.tx.BitmapNames()
+	if err != nil {
+		return nil, err
+	}
+
+	// Iterate over shard names and collect shards from matching field/view prefix.
+	for _, name := range names {
+		if !strings.HasPrefix(name, prefix) {
+			continue
+		}
+
+		s := strings.TrimPrefix(name, prefix)
+		shard, err := strconv.ParseUint(s, 10, 64)
+		if err != nil {
+			return nil, errors.Wrap(err, "parse shard id from rbf key")
+		}
+		sliceOfShards = append(sliceOfShards, shard)
+	}
+	return sliceOfShards, nil
+}
+
+func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
+	b, err := tx.RoaringBitmap(index, field, view, shard)
+	panicOn(err)
+	return b.Iterator()
+}
+
+func (tx *RBFTx) Pointer() string {
+	return fmt.Sprintf("%p", tx)
+}
+
+// Readonly is true if the transaction is not read-and-write, but only doing reads.
+func (tx *RBFTx) Readonly() bool {
+	return !tx.tx.Writable()
+}
+
+func (tx *RBFTx) UseRowCache() bool {
+	return false
+}
+
+// rbfName returns a NULL-separated key used for identifying bitmap maps in RBF.
+func rbfName(field, view string, shard uint64) string {
+	return fmt.Sprintf("%s\x00%s\x00%d", field, view, shard)
+}
+
+// rbfFieldViewPrefix returns a NULL-separated prefix for keys in RBF.
+func rbfFieldViewPrefix(field, view string) string {
+	return fmt.Sprintf("%s\x00%s\x00", field, view)
+}
diff --git a/txfactory.go b/txfactory.go
index 0f49982ec..99c80dd54 100644
--- a/txfactory.go
+++ b/txfactory.go
@@ -81,6 +81,8 @@ func (f *TxFactory) Store() TxStore {
 		//	case blueGreenRoaringBadger:
 	}
 	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
+=======
+>>>>>>> Implement pilosa.Tx for RBF
 }
 */
 
@@ -168,6 +170,7 @@ func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
 		typeOfTx:  ty,
 		roaringDB: NewRoaringStore(),
 	}
+
 	switch ty {
 	case badgerTxn, blueGreenBadgerRoaring, blueGreenRoaringBadger, blueGreenBadgerRBF, blueGreenRBFBadger:
 
@@ -193,10 +196,9 @@ func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
 
 	switch ty {
 	case rbfTxn, blueGreenRBFRoaring, blueGreenRoaringRBF, blueGreenBadgerRBF, blueGreenRBFBadger:
-		path := dir + sep + name + ".rbf"
-		f.rbfDB = rbf.NewDB(path)
+		f.rbfDB = rbf.NewDB(filepath.Join(dir, "db.rbf"))
 		if err := f.rbfDB.Open(); err != nil {
-			return nil, errors.Wrap(err, fmt.Sprintf("cannot open rbf db. path='%v'", path))
+			return nil, errors.Wrap(err, "cannot open rbf db")
 		}
 	}
 
@@ -240,8 +242,16 @@ func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uin
 	case badgerTxn:
 		return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
 	case rbfTxn:
-		//return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
-		return nil
+		tx, err := f.rbfDB.Begin(true)
+		if err != nil {
+			return err
+		}
+		defer tx.Rollback()
+
+		if err := tx.DeleteBitmapsWithPrefix(rbfFieldViewPrefix(field, view)); err != nil {
+			return err
+		}
+		return tx.Commit()
 	case blueGreenBadgerRoaring:
 		_ = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
 		return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
@@ -263,9 +273,7 @@ func (f *TxFactory) CloseIndex(idx *Index) error {
 		//return f.badgerDB.Close()
 		return nil
 	case rbfTxn:
-		// for same reason as above may not be able to close here.
-		//return f.rbfDB.Close()
-		return nil
+		return f.rbfDB.Close()
 	case blueGreenBadgerRoaring:
 		return nil
 	case blueGreenRoaringBadger:
@@ -275,7 +283,6 @@ func (f *TxFactory) CloseIndex(idx *Index) error {
 }
 
 func (f *TxFactory) NewTx(o Txo) Tx {
-
 	indexName := ""
 	if o.Index != nil {
 		indexName = o.Index.name
@@ -288,14 +295,11 @@ func (f *TxFactory) NewTx(o Txo) Tx {
 		btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
 		return btx
 	case rbfTxn:
-		panic("todo rbfTxn creation")
-		/*
-			rbftx, err := f.rbfDB.Begin(o.Write)
-			if err != nil {
-				errors.Wrap(err, "rbfDB.Begin transaction errored")
-			}
-			return rbftx
-		*/
+		tx, err := f.rbfDB.Begin(o.Write)
+		if err != nil {
+			panic(err) // TODO: Add error return on NewTx()
+		}
+		return &RBFTx{tx: tx}
 	case blueGreenBadgerRoaring:
 		btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
 		rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
@@ -379,7 +383,6 @@ func (idx *Index) StringifiedBadgerKeys(optionalUseThisTx Tx) string {
 // index directory, not including the name of the index itself.
 // The path should not start with the path separator sep ('/' or '\\') rune.
 func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, err error) {
-
 	if len(path) == 0 {
 		err = fmt.Errorf("fragmentSpecFromRoaringPath error: path '%v' too short", path)
 		return
@@ -408,7 +411,6 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64,
 }
 
 func (idx *Index) StringifiedRoaringKeys() (r string) {
-
 	paths, err := listFilesUnderDir(idx.path, false, "", true)
 	panicOn(err)
 	index := idx.name
diff --git a/xrbrsupport.go b/xrbrsupport.go
index ebfd9db27..70720582a 100644
--- a/xrbrsupport.go
+++ b/xrbrsupport.go
@@ -63,7 +63,7 @@ func (rbc *RBFConverter) Convert(index, field, view string, shard uint64, rb *ro
 	if err != nil {
 		return err
 	}
-	defer func() { _ = tx.Rollback() }()
+	defer tx.Rollback()
 
 	name := fmt.Sprintf("%s/%s", field, view)
 	err = tx.CreateBitmap(name)

From 55313bd69d8ad184b536b053d3b92f241aa0d5bb Mon Sep 17 00:00:00 2001
From: Cody Soyland <cody@soyland.org>
Date: Wed, 29 Jul 2020 15:28:58 -0500
Subject: [PATCH 10/14] Add CI job to ensure go mod files are tidy.

---
 .circleci/config.yml | 10 ++++++++++
 go.mod               |  7 +------
 go.sum               | 36 +++++++-----------------------------
 3 files changed, 18 insertions(+), 35 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 42de1db0c..6ecff4418 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -55,6 +55,13 @@ jobs:
       - checkout-plus
       - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.23.8
       - run: make golangci-lint
+  go-mod-tidy:
+    executor:
+      name: golang
+    steps:
+      - checkout-plus
+      - run: go mod tidy
+      - run: git diff --exit-code -- go.mod go.sum
   test-build-arm:
     executor:
       name: golang
@@ -168,6 +175,9 @@ workflows:
       - check-license-headers:
           requires:
             - setup
+      - go-mod-tidy:
+          requires:
+            - setup
       - test-build-arm:
           requires:
             - setup
diff --git a/go.mod b/go.mod
index acc83bb52..769fd50b9 100644
--- a/go.mod
+++ b/go.mod
@@ -9,11 +9,8 @@ require (
 	github.com/benbjohnson/immutable v0.2.0
 	github.com/boltdb/bolt v1.3.1
 	github.com/cespare/xxhash v1.1.0
-	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/davecgh/go-spew v1.1.1
-	github.com/dchest/blake2b v1.0.0 // indirect
-	github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85 // indirect
 	github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.2.0
@@ -22,8 +19,7 @@ require (
 	github.com/gorilla/handlers v1.3.0
 	github.com/gorilla/mux v1.7.0
 	github.com/hashicorp/memberlist v0.1.3
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2
+	github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 // indirect
 	github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/pelletier/go-toml v1.2.0
@@ -40,7 +36,6 @@ require (
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-client-go v2.16.0+incompatible
 	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
-	github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc // indirect
 	github.com/zeebo/blake3 v0.0.4
 	go.uber.org/atomic v1.4.0 // indirect
 	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
diff --git a/go.sum b/go.sum
index fbed4774f..5d09bde2d 100644
--- a/go.sum
+++ b/go.sum
@@ -26,9 +26,6 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
@@ -40,18 +37,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dchest/blake2b v1.0.0 h1:KK9LimVmE0MjRl9095XJmKqZ+iLxWATvlcpVFRtaw6s=
-github.com/dchest/blake2b v1.0.0/go.mod h1:U034kXgbJpCle2wSk5ybGIVhOSHCVLMDqOzcPEA0F7s=
-github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85 h1:oEqDRoxpep5ZlTxrAFc2yg+f0uBdUtkZE0uWsOru5bc=
-github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85/go.mod h1:cEjdIw+iaGXuQdsDymXPRcpp8yHXZ6PmwmDJajnVyJc=
-github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg=
 github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A=
 github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE=
-github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI=
-github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM=
-github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs=
-github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU=
-github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
 github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA=
 github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
@@ -115,8 +102,10 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -145,7 +134,6 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
 github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -180,29 +168,25 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
-github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
 github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
@@ -211,15 +195,13 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
 github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
 github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc h1:Jsemerl8qK30jGNdYlxGZpZk9RjB4pqvezJgxqUgy30=
-github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc/go.mod h1:9WHA/f8A/TRK+WQQZhqx47In4pnIhMTH6UrsgqqgsVQ=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0=
+github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
 github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
-github.com/zeebo/blake3 v0.0.4-0.20200428182842-252974700486 h1:yh0zEy8it58x/IPNtKKuvKUkxSIaq4s5XiRSd40JuYs=
-github.com/zeebo/blake3 v0.0.4-0.20200428182842-252974700486/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
 github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI=
 github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
+github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05 h1:4pW5fMvVkrgkMXdvIsVRRTs69DWYA8uNNQsu1stfVKU=
 github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -243,8 +225,6 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
-golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -263,8 +243,6 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8=
-golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
@@ -292,6 +270,7 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
@@ -302,4 +281,3 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
-vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130=

From 72c893a3d11fe7dd57ffed0c1cb049b79d15d2d8 Mon Sep 17 00:00:00 2001
From: Jason Aten <jason.aten@molecula.com>
Date: Tue, 28 Jul 2020 11:28:50 -0400
Subject: [PATCH 11/14] blueGreenTx roaring vs badger is all tests green (atg).

back to github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361  b/c github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200718033852-37ee16d8ad1c had issues with CI on 386 and arm
---
 Makefile                         |  60 +++----
 badger.go                        |  78 ++++++---
 badger_test.go                   |   2 +-
 bluegreentx.go                   | 280 ++++++++++++++++++++++++-------
 bluegreentx_test.go              |  91 ++++++++++
 catcher.go                       |   4 +
 cluster_internal_test.go         |   1 -
 executor_test.go                 |  31 ++--
 field_internal_test.go           |   9 +-
 fragment_internal_test.go        |  66 ++++----
 holder_test.go                   |  22 +--
 index.go                         |  15 +-
 roaring/roaring_internal_test.go |  37 ++++
 tx.go                            |  44 ++++-
 txfactory.go                     | 196 +++++++++++++++++-----
 utils_internal_test.go           |   1 +
 view.go                          |   7 +-
 vprint.go                        |  24 +++
 18 files changed, 745 insertions(+), 223 deletions(-)
 create mode 100644 bluegreentx_test.go

diff --git a/Makefile b/Makefile
index b8454402f..e794990ee 100644
--- a/Makefile
+++ b/Makefile
@@ -186,41 +186,43 @@ topt-race:
 
 # blue-green checks. These run two different storage engines (rbf, roaring, or badger)
 # and compare each transaction for a result.
-bg-br:
-	mv log.bg.bg_roar log.bg.bg_roar.prev || true
-	PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.bg_roar
-	@echo "   log.bg.bg_roar green: \c"; cat log.bg.bg_roar | grep PASS |wc -l
-	@echo "   log.bg.bg_roar   red: \c"; cat log.bg.bg_roar | grep '\-\-\- FAIL' |wc -l
 
-bg-rb:
+bg-rr: # shorthand for bluegreen test with A:badger; B:roaring
+	mv log.bg-rr log.bg-rr.prev || true
+	set -o pipefail; PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg-rr
+	@echo "   log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l
+	@echo "   log.bg-rr   red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l
+
+rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned).
 	mv log.bg.roar_bg log.bg.roar_bg.prev || true
-	PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.roar_bg
-	@echo "   log.bg.roar_bg green: \c"; cat log.bg.roar_bg | grep PASS |wc -l
-	@echo "   log.bg.roar_bg   red: \c"; cat log.bg.roar_bg | grep '\-\-\- FAIL' |wc -l
+	set -o pipefail; PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-bg
+	##PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-bg
+	@echo "   log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l
+	@echo "   log.rr-bg   red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l
 
-bg-fr:
-	mv log.bg.rbf_roar log.bg.rbf_roar.prev || true
-	PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.rbf_roar
-	@echo "   log.bg.rbf_roar green: \c"; cat log.bg.rbf_roar | grep PASS |wc -l
-	@echo "   log.bg.rbf_roar   red: \c"; cat log.bg.rbf_roar | grep '\-\-\- FAIL' |wc -l
+rbf-rr:
+	mv log.rbf-rr log.rbf-rr.prev || true
+	set -o pipefail; PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-rr
+	@echo "   log.rbf-rr green: \c"; cat log.rbf-rr | grep PASS |wc -l
+	@echo "   log.rbf-rr   red: \c"; cat log.rbf-rr | grep '\-\-\- FAIL' |wc -l
 
-bg-rf:
-	mv log.bg.roar_rbf log.bg.roar_rbf.prev || true
-	PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.roar_rbf
-	@echo "   log.bg.roar_rbf green: \c"; cat log.bg.roar_rbf | grep PASS |wc -l
-	@echo "   log.bg.roar_rbf   red: \c"; cat log.bg.roar_rbf | grep '\-\-\- FAIL' |wc -l
+rr-rbf:
+	mv log.rr-rbf log.rr-rbf.prev || true
+	set -o pipefail; PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rr-rbf
+	@echo "   log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l
+	@echo "   log.rr-rbf   red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l
 
-bg-fb:
-	mv log.bg.rbf_badger log.bg.rbf_badger.prev || true
-	PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.rbf_badger
-	@echo "   log.bg.rbf_badger green: \c"; cat log.bg.rbf_badger | grep PASS |wc -l
-	@echo "   log.bg.rbf_badger   red: \c"; cat log.bg.rbf_badger | grep '\-\-\- FAIL' |wc -l
+rbf-bg:
+	mv log.rbf-bg log.rbf-bg.prev || true
+	set -o pipefail; PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.rbf-bg
+	@echo "   log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l
+	@echo "   log.rbf-bg   red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l
 
-bg-bf:
-	mv log.bg.badger_rbf log.bg.badger_rbf.prev || true
-	PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg.badger_rbf
-	@echo "   log.bg.badger_rbf green: \c"; cat log.bg.badger_rbf | grep PASS |wc -l
-	@echo "   log.bg.badger_rbf   red: \c"; cat log.bg.badger_rbf | grep '\-\-\- FAIL' |wc -l
+bg-rbf:
+	mv log.bg-rbf log.bg-rbf.prev || true
+	set -o pipefail; PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.bg-rbf
+	@echo "   log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l
+	@echo "   log.bg-rbf   red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l
 
 
 # Run golangci-lint
diff --git a/badger.go b/badger.go
index b426e630e..f2fca4dc6 100644
--- a/badger.go
+++ b/badger.go
@@ -432,6 +432,8 @@ type BadgerDBWrapper struct {
 	// stack() from our creation point, to track tests
 	// that haven't closed us.
 	startStack string
+
+	DeleteEmptyContainer bool
 }
 
 // unprotectedListOpenTxAsString is a debugging helper.
@@ -477,12 +479,13 @@ func (w *BadgerDBWrapper) NewBadgerTx(write bool, initialIndexName string) (tx *
 	defer w.muDb.Unlock()
 
 	tx = &BadgerTx{
-		write:            write,
-		tx:               w.db.NewTransaction(write),
-		Db:               w,
-		initloc:          stack(),
-		doAllocZero:      w.doAllocZero,
-		initialIndexName: initialIndexName,
+		write:                write,
+		tx:                   w.db.NewTransaction(write),
+		Db:                   w,
+		initloc:              stack(),
+		doAllocZero:          w.doAllocZero,
+		initialIndexName:     initialIndexName,
+		DeleteEmptyContainer: w.DeleteEmptyContainer,
 	}
 
 	if w.openTx == nil {
@@ -541,6 +544,8 @@ type BadgerTx struct {
 	ourContainers []*roaring.Container
 
 	initialIndexName string
+
+	DeleteEmptyContainer bool
 }
 
 func (tx *BadgerTx) Type() string {
@@ -771,6 +776,11 @@ func badgerIndexOnlyPrefix(indexName string) []byte {
 	return []byte(fmt.Sprintf("idx:'%v';", indexName))
 }
 
+// same for deleting a whole field.
+func badgerFieldPrefix(index, field string) []byte {
+	return []byte(fmt.Sprintf("idx:'%v';fld:'%v';", index, field))
+}
+
 // Container returns the requested roaring.Container, selected by fragment and ckey
 func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) {
 
@@ -1012,7 +1022,10 @@ func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, f
 	if !bi.it.ValidForPrefix(prefix) {
 		return bi, false, nil
 	}
-	return bi, true, nil
+	item := bi.it.Item()
+	// have to compare b/c badger might give us valid iterator
+	// that is past our needle if needle isn't present.
+	return bi, bytes.Equal(item.Key(), needle), nil
 }
 
 // BadgerIterator is the iterator returned from a BadgerTx.ContainerIterator() call.
@@ -1301,18 +1314,18 @@ func (tx *BadgerTx) UnionInPlace(index, field, view string, shard uint64, others
 }
 
 // CountRange returns the count of hot bits in the start, end range on the fragment.
+// roaring.countRange counts the number of bits set between [start, end).
 func (tx *BadgerTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
 
 	skey := highbits(start)
 	ekey := highbits(end)
 
 	citer, found, err := tx.ContainerIterator(index, field, view, shard, skey)
+	_ = found
 	panicOn(err)
 
-	defer citer.Close() // doesn't seem to be getting called.
-	if !found {
-		return 0, nil
-	}
+	defer citer.Close()
+
 	// If range is entirely in one container then just count that range.
 	if skey == ekey {
 		citer.Next()
@@ -1485,7 +1498,7 @@ func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, i
 				changed += changes
 				rowSet[currRow] -= changes
 
-				if newC.N() == 0 {
+				if tx.DeleteEmptyContainer && newC.N() == 0 {
 					err = tx.RemoveContainer(index, field, view, shard, itrKey)
 					if err != nil {
 						return
@@ -1565,6 +1578,10 @@ const (
 
 func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
 
+	if len(v) == 0 {
+		return nil
+	}
+
 	// For safety we copy v, since it lives in BadgerDB's memory-mapped vlog-file,
 	// and Badger will recycle it after tx ends with rollback or commit.
 	// We copy into Go runtime GC managed memory. Technically we don't need
@@ -1609,30 +1626,28 @@ func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
 
 // fromArray16 converts to an 8KB page
 func fromArray16(a []uint16) []byte {
+	if len(a) == 0 {
+		return []byte{}
+	}
 	return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2]
 }
 
 // fromArray64 converts to an 8KB page
 func fromArray64(a []uint64) []byte {
+	if len(a) == 0 {
+		return []byte{}
+	}
 	return (*[8192]byte)(unsafe.Pointer(&a[0]))[:8192:8192]
 }
 
 // fromInterval16 converts to 8KB page
 func fromInterval16(a []roaring.Interval16) []byte {
+	if len(a) == 0 {
+		return []byte{}
+	}
 	return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4]
 }
 
-// badgerKey method on fragment creates a query key in the
-// standard format by invoking the top level badgerKey with
-// the container key being highbits(rowID * ShardWidth).
-//
-// Commented out for now only to keep the golangci-lint happy,
-// as it has no users at the moment.
-//func (f *fragment) badgerKey(rowID uint64) []byte {
-//	hi0 := highbits(rowID * ShardWidth)
-//	return badgerKey(f.index, f.field, f.view, f.shard, hi0)
-//}
-
 // StringifiedBadgerKeys returns a string with all the container
 // keys available in badger.
 func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx) (r string) {
@@ -1674,6 +1689,10 @@ func (tx *BadgerTx) countBitsSet(bkey []byte) (n int) {
 	return
 }
 
+func (tx *BadgerTx) Dump() {
+	fmt.Printf("%v\n", stringifiedBadgerKeysTx(tx))
+}
+
 // stringifiedBadgerKeysTx reports all the badger keys and a
 // corresponding blake3 hash viewable by txn within the entire
 // badger database.
@@ -1826,6 +1845,19 @@ func dirAsString(path string) (r string) {
 
 var _ = dirAsString // happy linter
 
+func (w *BadgerDBWrapper) DeleteField(index, field, fieldPath string) error {
+
+	// under blue-green roaring_badger, the directory will not be found, b/c roaring will have
+	// already done the os.RemoveAll().	BUT, RemoveAll returns nil error in this case. Docs:
+	// "If the path does not exist, RemoveAll returns nil (no error)"
+	err := os.RemoveAll(fieldPath)
+	if err != nil {
+		return errors.Wrap(err, "removing directory")
+	}
+	prefix := badgerFieldPrefix(index, field)
+	return w.DeletePrefix(prefix)
+}
+
 func (w *BadgerDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
 	prefix := badgerPrefix(index, field, view, shard)
 	return w.DeletePrefix(prefix)
diff --git a/badger_test.go b/badger_test.go
index 10fc0baa0..821356867 100644
--- a/badger_test.go
+++ b/badger_test.go
@@ -1052,6 +1052,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
 	index, field, view, shard := "i", "f", "v", uint64(0)
 	tx := dbwrap.NewBadgerTx(writable, index)
 	defer tx.Rollback()
+	tx.DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring.
 
 	//bitvalue := uint64(42)
 
@@ -1483,7 +1484,6 @@ func TestBadger_DeleteFragment(t *testing.T) {
 	}
 	err := tx.Commit()
 	panicOn(err)
-	//vv("Dump: %v", dbwrap.StringifiedBadgerKeys(nil))
 
 	// end of setup
 
diff --git a/bluegreentx.go b/bluegreentx.go
index 3df87577a..9df2596a1 100644
--- a/bluegreentx.go
+++ b/bluegreentx.go
@@ -15,27 +15,43 @@
 package pilosa
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"reflect"
 	"sort"
+	"sync"
 
 	"github.com/pilosa/pilosa/v2/roaring"
 )
 
 // blueGreenTx runs two Tx together and notices differences in their output.
 // By convention, the 'b' Tx is the output that is returned to caller.
+//
+// Warning: DATA RACES are expected if RoaringTx is one side of the Tx pair.
+// The checkDatabase() call will do reads of the fragments at Commit/Rollback,
+// while the snapshotqueue may be doing writes.
+//
+// Do not run with go test -race and expect it to be race free.
+//
 type blueGreenTx struct {
 	a Tx
 	b Tx // b's output is returned
 
+	as string
+	bs string
+
 	idx *Index
 
-	checker blueGreenChecker
+	checker              blueGreenChecker
+	mu                   sync.Mutex
+	rollbackOrCommitDone bool
 }
 
 func newBlueGreenTx(a, b Tx, idx *Index) *blueGreenTx {
-	return &blueGreenTx{a: a, b: b, idx: idx}
+	as := a.Type()
+	bs := b.Type()
+	return &blueGreenTx{a: a, b: b, idx: idx, as: as, bs: bs}
 }
 
 var _ = newBlueGreenTx // keep linter happy
@@ -46,6 +62,18 @@ func (c *blueGreenTx) Type() string {
 	return c.a.Type() + "_" + c.b.Type()
 }
 
+var blueGreenTxDumpMut sync.Mutex
+
+func (c *blueGreenTx) Dump() {
+	blueGreenTxDumpMut.Lock()
+	defer blueGreenTxDumpMut.Unlock()
+	fmt.Printf("%v blueGreenTx.Dump ============== \n", FileLine(2))
+	fmt.Printf("A(%v) Dump:\n", c.as)
+	c.a.Dump()
+	fmt.Printf("B(%v) Dump:\n", c.bs)
+	c.b.Dump()
+}
+
 func (c *blueGreenTx) Readonly() bool {
 	a := c.a.Readonly()
 	b := c.b.Readonly()
@@ -57,6 +85,7 @@ func (c *blueGreenTx) Readonly() bool {
 
 func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
 	c.checker.see(index, field, view, shard)
+	// TODO(jea): does this need to be different, to handle c.a iteration at the same time?
 	return c.b.NewTxIterator(index, field, view, shard)
 }
 
@@ -70,53 +99,70 @@ func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, chan
 	c.b.IncrementOpN(index, field, view, shard, changedN)
 }
 
+// compareTxState is called for the first Commit or Rollback a blueGreenTx sees.
 func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
 	here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard)
 	aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0)
 	bIter, bFound, bErr := c.b.ContainerIterator(index, field, view, shard, 0)
-
-	if aFound != bFound {
-		panic(fmt.Sprintf("compareTxState[%v]: A ContainerIterator had aFound=%v, but B had bFound=%v; at '%v'", here, aFound, bFound, stack()))
-	}
-
-	if aErr == nil {
+	if aErr == nil || aIter != nil {
 		defer aIter.Close()
 	}
-	if bErr == nil {
+	if bErr == nil || bIter != nil {
 		defer bIter.Close()
 	}
+
+	if aFound != bFound {
+		c.Dump()
+		panic(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, stack()))
+	}
+
 	if aErr != nil || bErr != nil {
 		if aErr != nil && bErr != nil {
-			panic(fmt.Sprintf("compareTxState[%v]: A reported err '%v'; B reported err '%v' at %v", here, aErr, bErr, stack()))
+			c.Dump()
+			panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, stack()))
 		}
 		if aErr != nil {
-			panic(fmt.Sprintf("compareTxState[%v]: A reported err %v at %v; but B did not", here, aErr, stack()))
+			c.Dump()
+			panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, stack()))
 		}
 		if bErr != nil {
-			panic(fmt.Sprintf("compareTxState[%v]: B reported err %v at %v; but A did not", here, bErr, stack()))
+			c.Dump()
+			panic(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, stack()))
 		}
 	}
+
 	for aIter.Next() {
 		aKey, aValue := aIter.Value()
+
 		if !bIter.Next() {
-			panic(fmt.Sprintf("compareTxState[%v]: A found key %v, B didn't, at %v", here, aKey, stack()))
+			c.Dump()
+			panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, stack()))
 		}
 		bKey, bValue := bIter.Value()
 		if bKey != aKey {
-			panic(fmt.Sprintf("compareTxState[%v]: A found key %v, B found %v, at %v", here, aKey, bKey, stack()))
+			AlwaysPrintf("problem in caller %v", Caller(2))
+			c.Dump()
+			panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack())) // crashing here on TestBSIGroup_importValue
 		}
 		if err := aValue.BitwiseCompare(bValue); err != nil {
-			panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v at %v", here, aKey, err, stack()))
+			c.Dump()
+			panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v;  A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack()))
 		}
 	}
 	// end checking everything in A, but does B have more?
 	if bIter.Next() {
 		bKey, _ := bIter.Value()
-		panic(fmt.Sprintf("compareTxState[%v]: B found key %v, A didn't, at %v", here, bKey, stack()))
+		c.Dump()
+		panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack()))
 	}
 }
 
 func (c *blueGreenTx) checkDatabase() {
+	c.checker.mu.Lock()
+	defer c.checker.mu.Unlock()
+
+	// seen() returns nil on 2nd or any further call,
+	// so only the first Commit() or Rollback() does this.
 	for index, fields := range c.checker.seen() {
 		for field, views := range fields {
 			for view, shards := range views {
@@ -129,6 +175,13 @@ func (c *blueGreenTx) checkDatabase() {
 }
 
 func (c *blueGreenTx) Rollback() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.rollbackOrCommitDone {
+		return
+	}
+	c.rollbackOrCommitDone = true
+
 	c.checkDatabase()
 	defer func() {
 		if r := recover(); r != nil {
@@ -141,6 +194,12 @@ func (c *blueGreenTx) Rollback() {
 }
 
 func (c *blueGreenTx) Commit() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.rollbackOrCommitDone {
+		return nil
+	}
+	c.rollbackOrCommitDone = true
 	c.checkDatabase()
 	defer func() {
 		if r := recover(); r != nil {
@@ -168,6 +227,13 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r
 	_, _ = a, errA
 	b, errB := c.b.RoaringBitmap(index, field, view, shard)
 	compareErrors(errA, errB)
+
+	slcA := a.Slice()
+	slcB := b.Slice()
+	if !reflect.DeepEqual(slcA, slcB) {
+		panic("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!")
+	}
+
 	return b, errB
 }
 
@@ -205,6 +271,14 @@ func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key
 
 func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
 	c.checker.see(index, field, view, shard)
+
+	// these are the first port of call for debugging, so we leave them in.
+	// ================== begin save comments.
+	//c.checkDatabase()
+	//vv("got past database check at TOP of ImportRoaringBits")
+	//c.Dump()
+	//vv("done with top dump; clear=%v", clear)
+	// ==================   end save comments.
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
@@ -214,9 +288,9 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
 
 	// remember where the iterator started, so we can replay it a second time.
 	rit2 := rit.Clone()
+	panicOn(err)
 
 	changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
-
 	changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data)
 
 	if len(data) == 0 {
@@ -240,7 +314,7 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
 		}
 	}
 	compareErrors(errA, errB)
-
+	c.checkDatabase()
 	return changedB, rowSetB, errB
 }
 
@@ -344,50 +418,101 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64,
 			panic(r)
 		}
 	}()
-	// TODO: need to return a blueGreenIterator too, that does close/next operations on both A and B.
+
 	ait, afound, errA := c.a.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
 	_, _, _ = ait, afound, errA
+
 	bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
 
 	compareErrors(errA, errB)
-	if errA != nil {
-		ait.Close() // don't leak it.
+	// INVAR: errA == errB, so only need to check one.
+	if errB != nil {
+		// RoaringTx can return an iterator and an error, so be sure Close it we have it.
+		if ait != nil {
+			ait.Close()
+		}
+		if bit != nil {
+			bit.Close()
+		}
+		return nil, bfound, errB
 	}
-	return bit, bfound, errB
+	// INVAR: errA == errB == nil
+	bgi := NewBlueGreenIterator(c, ait, bit)
+	return bgi, bfound, errB
 }
 
+func NewBlueGreenIterator(tx *blueGreenTx, ait, bit roaring.ContainerIterator) *blueGreenIterator {
+	return &blueGreenIterator{
+		tx:  tx,
+		as:  tx.as,
+		bs:  tx.bs,
+		ait: ait,
+		bit: bit,
+	}
+}
+
+type blueGreenIterator struct {
+	tx *blueGreenTx
+	as string
+	bs string
+
+	ait roaring.ContainerIterator
+	bit roaring.ContainerIterator
+}
+
+func (bgi *blueGreenIterator) Next() bool {
+	na := bgi.ait.Next()
+	nb := bgi.bit.Next()
+	if na != nb {
+		panic(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb))
+	}
+	return nb
+}
+
+func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) {
+	ka, ca := bgi.ait.Value()
+	kb, cb := bgi.bit.Value()
+	if ka != kb {
+		panic(fmt.Sprintf("ka=%v != kb=%v", ka, kb))
+	}
+	err := ca.BitwiseCompare(cb)
+	panicOn(err)
+	return kb, cb
+}
+func (bgi *blueGreenIterator) Close() {
+	bgi.ait.Close()
+	bgi.bit.Close()
+}
+
+// ForEach is read-only on the database, and so we only pass through to B.
+// Avoids the side-effects of calling fn too many times.
 func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
-	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
 			panic(r)
 		}
 	}()
-	errA := c.a.ForEach(index, field, view, shard, fn)
-	_ = errA
-	errB := c.b.ForEach(index, field, view, shard, fn)
-	_ = errB
+	return c.b.ForEach(index, field, view, shard, fn)
 
-	compareErrors(errA, errB)
-	return errB
 }
 
+// ForEachRange cannot change the database, and we also can't control
+// the side effects of the fn() calls. So we only pass through to B, not A.
+// No checker.see() is needed as well, because we are read-only.
 func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
-	c.checker.see(index, field, view, shard)
+
 	defer func() {
 		if r := recover(); r != nil {
 			AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
 			panic(r)
 		}
 	}()
-	errA := c.a.ForEachRange(index, field, view, shard, start, end, fn)
-	_ = errA
-	errB := c.b.ForEachRange(index, field, view, shard, start, end, fn)
-	_ = errB
 
-	compareErrors(errA, errB)
-	return errB
+	// calling fn will have side effects; can only call it the right number of times.
+	// so can't do this.
+	//	errA := c.a.ForEachRange(index, field, view, shard, start, end, fn)
+	return c.b.ForEachRange(index, field, view, shard, start, end, fn)
 }
 
 func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, error) {
@@ -459,6 +584,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
 	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
+			c.Dump()
 			AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
 			panic(r)
 		}
@@ -467,7 +593,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
 	b, errB := c.b.CountRange(index, field, view, shard, start, end)
 
 	if a != b {
-		panic(fmt.Sprintf("a = %v, but b = %v", a, b))
+		panic(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b))
 	}
 
 	compareErrors(errA, errB)
@@ -495,18 +621,35 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6
 	c.checker.see(index, field, view, shard)
 	defer func() {
 		if r := recover(); r != nil {
-			AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
+			c.Dump()
+			AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
 			panic(r)
 		}
 	}()
 
 	rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
 	rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
-	if szA != szB {
-		panic(fmt.Sprintf("szA = %v, but szB = %v", szA, szB))
-	}
+
 	compareErrors(errA, errB)
-	return &MultiReaderB{a: rcA, b: rcB}, szB, errB
+
+	// We are seeing Roaring vs Badger size differences on
+	// server/ test TestClusterResize_AddNode/ContinuousShards,
+	// so turn off the szA vs szB checks and MutliReaderB use. But keep them if we want to
+	// check RBF vs Badger for byte-for-byte compatiblity (we
+	// suspect the ops log or optimized bitmaps are accounting for the difference).
+	sizeMustMatch := false
+	if sizeMustMatch {
+		if szA != szB {
+			panic(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v;  fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring))
+		}
+		return &MultiReaderB{a: rcA, b: rcB}, szB, errB
+	} else {
+		// one db won't get data if we do
+		//return &MultiReaderB{a: rcA, b: rcB, allowSizeVariation: true}, szB, errB
+		_, _ = szA, errA
+		rcA.Close()
+		return rcB, szB, errB
+	}
 }
 
 func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
@@ -514,6 +657,7 @@ func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string)
 	//c.checker.see(index, field, view, shard) // don't have shard.
 	defer func() {
 		if r := recover(); r != nil {
+			c.Dump()
 			AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
 			panic(r)
 		}
@@ -536,30 +680,38 @@ func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string)
 		}
 		for _, kb := range slcB {
 			if !ma[kb] {
-				panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B had %v, but A did not; in the SliceOfShards returned slice.", kb))
+				c.Dump()
+				panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb))
 			}
 			delete(ma, kb)
 		}
 		if len(ma) != 0 {
-			for _, firstDifference := range ma {
-				panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A had %v, but B did not; in the SliceOfShards returned slice.", firstDifference))
+			for firstDifference := range ma {
+				panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A(%v) had %v, but B(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.as, firstDifference, c.bs, cpa, cpb))
 			}
 		}
-		panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA='%#v';\n slcB='%#v';\n", cpa, cpb))
+		panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA(%v)='%#v';\n slcB(%v)='%#v';\n", c.as, cpa, c.bs, cpb))
 	}
 	return slcB, errB
 }
 
+// MultiReaderB is returned by RoaringBitmapReader. It verifies
+// that identical byte streams are read from its two members.
 type MultiReaderB struct {
 	a io.ReadCloser
 	b io.ReadCloser
+
+	allowSizeVariation bool
 }
 
-// TODO(jea): test this for accuracy/correctness.
+// Read implements the standard io.Reader method. It panics
+// if "a" and "b" have even one byte different in their reads.
 func (m *MultiReaderB) Read(p []byte) (nB int, errB error) {
 	nB, errB = m.b.Read(p)
 	p2 := make([]byte, nB)
-	// discard the exact same amount from A
+
+	// read (and discard after comparing for equality) the exact same amount from A.
+
 	// ReadAtLeast reads from r into buf until it has read at least
 	// min bytes. It returns the number of bytes copied and an error
 	// if fewer bytes were read. The error is EOF only if no bytes
@@ -569,11 +721,18 @@ func (m *MultiReaderB) Read(p []byte) (nB int, errB error) {
 	// return, n >= min if and only if err == nil. If r returns
 	// an error having read at least min bytes, the error is dropped.
 	nA, errA := io.ReadAtLeast(m.a, p2, nB)
-	if errA == io.ErrUnexpectedEOF {
-		panic(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA))
-	}
-	if nA != nB {
-		panic(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA))
+
+	if !m.allowSizeVariation {
+		if errA == io.ErrUnexpectedEOF {
+			panic(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA))
+		}
+		if nA != nB {
+			panic(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA))
+		}
+		cmp := bytes.Compare(p[:nB], p2[:nB])
+		if cmp != 0 {
+			panic(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp)) // \np  ='%v'; \np2 ='%v'", cmp, string(p[:nB]), string(p2[:nA])))
+		}
 	}
 	return
 }
@@ -586,11 +745,20 @@ func (m *MultiReaderB) Close() error {
 // blueGreenChecker is used
 type blueGreenChecker struct {
 	visited map[string]map[string]map[string]map[uint64]struct{}
-	done    bool
+
+	// lock mu when using visited.
+	// otherwise concurrent map writes on TestAPI_Import/RowIDColumnKey
+	mu sync.Mutex
 }
 
 // see would mark a thing as seen.
 func (b *blueGreenChecker) see(index, field, view string, shard uint64) {
+	// keep this next Printf. Useful to see the sequence of Tx operations.
+	//fmt.Printf("blueGreenTx.%v\n", Caller(1))
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
 	if b.visited == nil {
 		b.visited = make(map[string]map[string]map[string]map[uint64]struct{})
 	}
@@ -617,9 +785,5 @@ func (b *blueGreenChecker) see(index, field, view string, shard uint64) {
 // that Rollback can be called after Commit without repeating
 // the check.
 func (b *blueGreenChecker) seen() map[string]map[string]map[string]map[uint64]struct{} {
-	if b.done {
-		return nil
-	}
-	b.done = true
 	return b.visited
 }
diff --git a/bluegreentx_test.go b/bluegreentx_test.go
new file mode 100644
index 000000000..131712450
--- /dev/null
+++ b/bluegreentx_test.go
@@ -0,0 +1,91 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"testing"
+
+	cryrand "crypto/rand"
+)
+
+func TestMultiReaderB(t *testing.T) {
+	// MultiReaderB should read identical chunks of bytes from both its "a" and "b"
+	// member io.Readers, else it should panic. This should hold for
+	// varying sizes of inputs.
+
+	for n := 1 << 5; n < (1 << 18); n = n*2 - 13 {
+		src := io.LimitReader(cryrand.Reader, int64(n))
+
+		a := make([]byte, n)
+		nr := 0
+		for nr < n {
+			na, err := src.Read(a)
+			panicOn(err)
+			nr += na
+		}
+		if nr != n {
+			panic("short read")
+		}
+
+		b := make([]byte, n)
+		copy(b, a)
+		if !bytes.Equal(a, b) {
+			panic("test prep failed")
+		}
+
+		m := &MultiReaderB{
+			a: ioutil.NopCloser(bytes.NewBuffer(a)),
+			b: ioutil.NopCloser(bytes.NewBuffer(b)),
+		}
+
+		// should not trigger the internal panic of MultiReadB
+		ncp, err := io.Copy(ioutil.Discard, m)
+		panicOn(err)
+		if ncp != int64(n) {
+			panic("short copy")
+		}
+
+		for victim := 0; victim < n; victim += 7 {
+
+			copy(b, a)
+			if victim%2 == 0 {
+				// corrupt b
+				b[victim] = (b[victim] + 1) % 255
+			} else {
+				// corrupt a
+				a[victim] = (a[victim] + 1) % 255
+			}
+			m = &MultiReaderB{
+				a: ioutil.NopCloser(bytes.NewBuffer(a)),
+				b: ioutil.NopCloser(bytes.NewBuffer(b)),
+			}
+			helperShouldPanicOnCopy(m)
+		}
+	}
+}
+
+func helperShouldPanicOnCopy(m *MultiReaderB) {
+	// differences in bytes read should be noticed
+	defer func() {
+		r := recover()
+		if r == nil {
+			panic("expected panic on byte difference but didn't see it")
+		}
+	}()
+	_, _ = io.Copy(ioutil.Discard, m)
+}
diff --git a/catcher.go b/catcher.go
index 5623733c2..2394eb331 100644
--- a/catcher.go
+++ b/catcher.go
@@ -62,6 +62,10 @@ func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, r
 	return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
 }
 
+func (c *catcherTx) Dump() {
+	c.b.Dump()
+}
+
 func (c *catcherTx) Readonly() bool {
 	defer func() {
 		if r := recover(); r != nil {
diff --git a/cluster_internal_test.go b/cluster_internal_test.go
index 1c344da06..32ceec7f7 100644
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@@ -854,7 +854,6 @@ func TestCluster_ResizeStates(t *testing.T) {
 			t.Fatal(err)
 		} else if !bytes.Equal(chksum, node0Checksum) {
 			t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
-			// badger red:  TestCluster_ResizeStates/Multiple_nodes,_with_data: cluster_internal_test.go:841: expected standard view checksum to match: ef46db3751d8e999 - fad4de25ee696ca0
 		}
 
 		// Close TestCluster.
diff --git a/executor_test.go b/executor_test.go
index b19c6cdf0..dd4927209 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -23,6 +23,7 @@ import (
 	"io/ioutil"
 	"math"
 	"math/rand"
+	"os"
 	"reflect"
 	"strconv"
 	"strings"
@@ -513,6 +514,12 @@ func TestExecutor_Execute_Count(t *testing.T) {
 
 }
 
+func roaringOnlyTest(t *testing.T) {
+	if os.Getenv("PILOSA_TXSRC") != "roaring" {
+		t.Skip("skip for everything but roaring")
+	}
+}
+
 // Ensure a set query can be executed.
 func TestExecutor_Execute_Set(t *testing.T) {
 	t.Run("RowIDColumnID", func(t *testing.T) {
@@ -521,7 +528,7 @@ func TestExecutor_Execute_Set(t *testing.T) {
 		cmd := cluster[0]
 		holder := cmd.Server.Holder()
 		hldr := test.Holder{Holder: holder}
-		hldr.SetBit("i", "f", 1, 0)
+		hldr.SetBit("i", "f", 1, 0) // creates and commits a Tx internally.
 
 		t.Run("OK", func(t *testing.T) {
 			hldr.ClearBit("i", "f", 11, 1)
@@ -582,10 +589,10 @@ func TestExecutor_Execute_Set(t *testing.T) {
 		cmd := cluster[0]
 		holder := cmd.Server.Holder()
 		hldr := test.Holder{Holder: holder}
-		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
+		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
 
 		t.Run("OK", func(t *testing.T) {
-			hldr.SetBit("i", "f", 1, 0)
+			hldr.SetBit("i", "f", 1, 0) // creates and Commits a Tx internally.
 			if n := hldr.Row("i", "f", 11).Count(); n != 0 {
 				t.Fatalf("unexpected row count: %d", n)
 			}
@@ -619,14 +626,16 @@ func TestExecutor_Execute_Set(t *testing.T) {
 		})
 
 		t.Run("ErrInvalidColValueType", func(t *testing.T) {
-			if err := index.DeleteField("f"); err != nil {
-				t.Fatal(err)
-			}
-			if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
+
+			if err := idx.DeleteField("f"); err != nil {
 				t.Fatal(err)
 			}
 
-			if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2.1, f=1)`}); err == nil || strings.Contains(err.Error(), `column value must be a string or non-negative integer`) {
+			if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
+				t.Fatal(err)
+			}
+
+			if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2.1, f=1)`}); err == nil || !strings.Contains(err.Error(), "parse error") {
 				t.Fatal(err)
 			}
 
@@ -637,9 +646,9 @@ func TestExecutor_Execute_Set(t *testing.T) {
 			}
 		})
 
-		t.Run("ErrInvalidRowValueType", func(t *testing.T) {
-			index := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{})
-			if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
+		t.Run("ErrInvalidRowValueType", func(t *testing.T) { // // failing under badger_roaring
+			idx := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{})
+			if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
 				t.Fatal(err)
 			}
 			if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "inokey", Query: `Set(2, f=1.2)`}); err == nil || !strings.Contains(err.Error(), "row value must be a string or non-negative integer") {
diff --git a/field_internal_test.go b/field_internal_test.go
index 529cabee8..0472e7dcc 100644
--- a/field_internal_test.go
+++ b/field_internal_test.go
@@ -611,22 +611,25 @@ func TestBSIGroup_importValue(t *testing.T) {
 		},
 	} {
 		tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
-		defer tx.Rollback()
+		// can't do this, we are in a loop, not a function:
+		// defer tx.Rollback()
 
 		if err := f.importValue(tx, tt.columnIDs, tt.values, options); err != nil {
 			t.Fatalf("test %d, importing values: %s", i, err.Error())
 		}
 
 		panicOn(tx.Commit())
+
 		tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
-		defer tx.Rollback()
+		// no, same reason as above: defer tx.Rollback()
 
 		if row, err := f.Range(tx, f.name, pql.EQ, tt.checkVal); err != nil {
 			t.Fatalf("test %d, getting range: %s", i, err.Error())
 		} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
 			t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
 		}
-	}
+		tx.Rollback()
+	} // loop
 }
 
 func TestIntField_MinMaxForShard(t *testing.T) {
diff --git a/fragment_internal_test.go b/fragment_internal_test.go
index c7de1a632..f30e27ef2 100644
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@@ -1771,6 +1771,8 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
 		t.Fatalf("unexpected cache len: %d", cache.Len())
 	}
 
+	panicOn(tx.Commit())
+
 	// Reopen the fragment.
 	if err := f.Reopen(); err != nil {
 		t.Fatal(err)
@@ -1847,16 +1849,27 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
 	}
 }
 
+func roaringOnlyTest(t *testing.T) {
+	if os.Getenv("PILOSA_TXSRC") != "roaring" {
+		t.Skip("skip for everything but roaring")
+	}
+}
+
+func roaringOnlyBenchmark(b *testing.B) {
+	if os.Getenv("PILOSA_TXSRC") != "roaring" {
+		b.Skip("skip for everything but roaring")
+	}
+}
+
 // Ensure a fragment can be copied to another fragment.
 func TestFragment_WriteTo_ReadFrom(t *testing.T) {
-	skipForRBF(t)
+	roaringOnlyTest(t)
 
 	f0, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
 	_ = idx
 	defer f0.Clean(t)
 
-	// Obtain transaction.
-	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f0})
+	tx := f0.txTestingOnly
 	defer tx.Rollback()
 
 	// Set and then clear bits on the fragment.
@@ -2032,6 +2045,8 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
 }
 
 func TestFragment_Snapshot_Run(t *testing.T) {
+	roaringOnlyTest(t)
+
 	f, idx := mustOpenFragment("i", "f", viewStandard, 0, "")
 	_ = idx
 	defer f.Clean(t)
@@ -3094,10 +3109,7 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) {
 	}
 }
 func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
-	skipForBadger := os.Getenv("PILOSA_TXSRC") == "badger"
-	if skipForBadger {
-		b.Skip("skip for badger")
-	}
+	roaringOnlyBenchmark(b)
 	if testing.Short() {
 		b.SkipNow()
 	}
@@ -3385,7 +3397,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) {
 		//nf, idx := mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte)
 
 		th := newTestHolder()
-		idx := fragTestMustOpenIndex("i", th, IndexOptions{})
+		idx := fragTestMustOpenIndex(filepath.Dir(fi.Name()), "i", th, IndexOptions{})
 		if th.NeedsSnapshot() {
 			th.SnapshotQueue = newSnapshotQueue(1, 1, nil)
 		}
@@ -3663,12 +3675,8 @@ func newTestHolder() *Holder {
 }
 
 // fragTestMustOpenIndex returns a new, opened index at a temporary path. Panic on error.
-func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Index {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
-	if err != nil {
-		panic(err)
-	}
-	holder.Path = path
+func fragTestMustOpenIndex(holderDir, index string, holder *Holder, opt IndexOptions) *Index {
+	holder.Path = holderDir
 	holder.mu.Lock()
 	idx, err := holder.createIndex(index, opt)
 	holder.mu.Unlock()
@@ -3685,23 +3693,24 @@ func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Inde
 
 // mustOpenFragment returns a new instance of Fragment with a temporary path.
 func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) (*fragment, *Index) {
-	file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-")
-	if err != nil {
-		panic(err)
-	}
-	file.Close()
+
+	holderDir, err := ioutil.TempDir(*TempDir, "holder-dir")
+	panicOn(err)
 
 	if cacheType == "" {
 		cacheType = DefaultCacheType
 	}
 
-	// new:
 	th := newTestHolder()
-	idx := fragTestMustOpenIndex(index, th, IndexOptions{})
+	idx := fragTestMustOpenIndex(holderDir, index, th, IndexOptions{})
 	if th.NeedsSnapshot() {
 		th.SnapshotQueue = newSnapshotQueue(1, 1, nil)
 	}
-	f := newFragment(th, file.Name(), index, field, view, shard, flags)
+
+	fragDir := fmt.Sprintf("%v/%v/views/%v/fragments/", idx.path, field, view)
+	panicOn(os.MkdirAll(fragDir, 0777))
+	fragPath := fragDir + fmt.Sprintf("%v", shard)
+	f := newFragment(th, fragPath, index, field, view, shard, flags)
 
 	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
 	f.txTestingOnly = tx
@@ -3811,6 +3820,10 @@ func TestFragment_RowsIteration(t *testing.T) {
 		} else if _, err := f.setBit(tx, 2, 166000); err != nil {
 			t.Fatal(err)
 		}
+		panicOn(tx.Commit())
+
+		tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
+		defer tx.Rollback()
 
 		ids, err := f.rows(context.Background(), tx, 0)
 		if err != nil {
@@ -4421,11 +4434,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
 }
 
 func TestUnionInPlaceMapped(t *testing.T) {
-
-	skipForBadger := os.Getenv("PILOSA_TXSRC") == "badger"
-	if skipForBadger {
-		t.Skip("skip for badger")
-	}
+	roaringOnlyTest(t)
 
 	f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone)
 	// note: clean has to be deferred first, because it has to run with
@@ -5112,6 +5121,8 @@ func TestFragmentBSISigned(t *testing.T) {
 }
 
 func TestImportClearRestart(t *testing.T) {
+	roaringOnlyTest(t)
+
 	tests := []struct {
 		rows []uint64
 		cols []uint64
@@ -5272,7 +5283,6 @@ func TestImportClearRestart(t *testing.T) {
 
 				err = f3.Open()
 				if err != nil {
-					// TODO(jea): might be a flaky test? when run from make test
 					t.Fatalf("opening f3: %v", err)
 				}
 				defer f3.Clean(t)
diff --git a/holder_test.go b/holder_test.go
index 2dfc55dc7..08b84acc6 100644
--- a/holder_test.go
+++ b/holder_test.go
@@ -32,8 +32,6 @@ import (
 )
 
 func TestHolder_Open(t *testing.T) {
-	skipForBadger := os.Getenv("PILOSA_TXSRC") == "badger"
-	skipForRBF := os.Getenv("PILOSA_TXSRC") == "rbf"
 
 	t.Run("ErrIndexName", func(t *testing.T) {
 		h := test.MustOpenHolder()
@@ -168,11 +166,8 @@ func TestHolder_Open(t *testing.T) {
 	})
 
 	t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
-		if skipForBadger {
-			t.Skip("skipping for badger")
-		} else if skipForRBF {
-			t.Skip("skipping for rbf")
-		}
+		roaringOnlyTest(t)
+
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
@@ -209,11 +204,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
-		if skipForBadger {
-			t.Skip("skipping for badger")
-		} else if skipForRBF {
-			t.Skip("skipping for rbf")
-		}
+		roaringOnlyTest(t)
 
 		h := test.MustOpenHolder()
 		defer h.Close()
@@ -247,11 +238,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
-		if skipForBadger {
-			t.Skip("skipping for badger")
-		} else if skipForRBF {
-			t.Skip("skipping for rbf")
-		}
+		roaringOnlyTest(t)
 
 		h := test.MustOpenHolder()
 		defer h.Close()
@@ -594,7 +581,6 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
 
 	// Leave the third replica empty to force a block merge.
 	//
-
 	err = c[0].Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
diff --git a/index.go b/index.go
index f9c6d2099..6007af4f8 100644
--- a/index.go
+++ b/index.go
@@ -621,9 +621,8 @@ func (i *Index) DeleteField(name string) error {
 		return errors.Wrap(err, "closing")
 	}
 
-	// Delete field directory.
-	if err := os.RemoveAll(i.fieldPath(name)); err != nil {
-		return errors.Wrap(err, "removing directory")
+	if err := i.Txf.DeleteFieldFromStore(i.name, name, i.fieldPath(name)); err != nil {
+		return errors.Wrap(err, "Txf.DeleteFieldFromStore")
 	}
 
 	// If the field being deleted is the existence field,
@@ -700,3 +699,13 @@ type importValueData struct {
 func FormatQualifiedIndexName(index string) string {
 	return fmt.Sprintf("%s\x00", index)
 }
+
+// Dump prints to stdout the contents of the roaring Containers
+// stored in idx. Mostly for debugging.
+func (idx *Index) Dump(label string) {
+	fileline := FileLine(2)
+	tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx})
+	defer tx.Rollback()
+	fmt.Printf("\n%v Index.Dump('%v') for index '%v':\n", fileline, label, idx.name)
+	tx.Dump()
+}
diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go
index 42b34860e..2041af9ff 100644
--- a/roaring/roaring_internal_test.go
+++ b/roaring/roaring_internal_test.go
@@ -4404,3 +4404,40 @@ func TestUnionRunRunInPlaceBitwiseCompare(t *testing.T) {
 		}
 	}
 }
+
+func TestCloneRoaringIterator(t *testing.T) {
+
+	ca := NewContainerArray([]uint16{1, 10, 100, 1000})
+	ba := NewFileBitmap()
+	ba.Containers.Put(0, ca)
+	ba.Containers.Put(10, ca)
+	ba.Containers.Put(101, ca)
+	ba.Containers.Put(10001, ca)
+	var buf bytes.Buffer
+	_, err := ba.WriteTo(&buf)
+	if err != nil {
+		t.Fatalf("error writing: %v", err)
+	}
+
+	itr, err := NewRoaringIterator(buf.Bytes())
+	if err != nil {
+		t.Fatalf("error NewRoaringIterator(buf.Bytes()): %v", err)
+	}
+
+	itr2 := itr.Clone()
+
+	var keys []uint64
+	for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() {
+		keys = append(keys, itrKey)
+		_ = synthC
+	}
+
+	var keys2 []uint64
+	for itrKey, synthC := itr2.NextContainer(); synthC != nil; itrKey, synthC = itr2.NextContainer() {
+		keys2 = append(keys2, itrKey)
+		_ = synthC
+	}
+	if !reflect.DeepEqual(keys, keys2) {
+		t.Fatalf("keys != keys2. keys='%#v'; keys2='%#v'", keys, keys2)
+	}
+}
diff --git a/tx.go b/tx.go
index 94fbbb383..051397245 100644
--- a/tx.go
+++ b/tx.go
@@ -198,9 +198,12 @@ type Tx interface {
 
 	// SliceOfShards returns all of the shards for the specified index, field, view triple.
 	// Use within pilosa supposes a new read-only transaction was created just
-	// for the SliceOfShards() call. The original Roaring version is the only
+	// for the SliceOfShards() call. The legacy RoaringTx version is the only
 	// one that needs optionalViewPath; any other Tx implementation can ignore that.
 	SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error)
+
+	// Dump is for debugging, what does this Tx see as its database?
+	Dump()
 }
 
 // TxStore has operations that will create and commit multiple
@@ -222,6 +225,8 @@ type TxStore interface {
 	//
 	DeleteFragment(index, field, view string, shard uint64, frag interface{}) error
 
+	DeleteField(index, field string) error
+
 	// Close shuts down the database.
 	Close() error
 }
@@ -269,6 +274,19 @@ func (mtx *MultiTx) Type() string {
 	return RoaringTxn
 }
 
+// debugging, what does this Tx see as its database?
+func (mtx *MultiTx) Dump() {
+	mtx.mu.Lock()
+	defer mtx.mu.Unlock()
+	if len(mtx.txs) == 0 {
+		return
+	}
+	for _, tx := range mtx.txs {
+		tx.Dump()
+		return
+	}
+}
+
 func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
 	tx, err := mtx.txNoShard(index)
 	panicOn(err)
@@ -516,10 +534,14 @@ type RoaringTx struct {
 	fragment *fragment
 }
 
-func (mtx *RoaringTx) Type() string {
+func (tx *RoaringTx) Type() string {
 	return RoaringTxn
 }
 
+func (tx *RoaringTx) Dump() {
+	fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys())
+}
+
 func (tx *RoaringTx) UseRowCache() bool {
 	return true
 }
@@ -548,6 +570,7 @@ func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string)
 		// Parse filename into integer.
 		shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
 		if err != nil {
+			//AlwaysPrintf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
 			//v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
 			continue
 		}
@@ -572,7 +595,6 @@ func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roa
 // the data []byte is supplied. This mimics the traditional roaring-per-file
 // and should be faster.
 func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
-
 	f, err := tx.getFragment(index, field, view, shard)
 	if err != nil {
 		return 0, nil, err
@@ -836,6 +858,18 @@ func (db *RoaringStore) Close() error {
 	return nil
 }
 
+func (db *RoaringStore) DeleteField(index, field, fieldPath string) error {
+
+	// under blue-green badger_roaring, the directory will not be found, b/c badger will have
+	// already done the os.RemoveAll().	BUT, RemoveAll returns nil error in this case. Docs:
+	// "If the path does not exist, RemoveAll returns nil (no error)"
+	err := os.RemoveAll(fieldPath)
+	if err != nil {
+		return errors.Wrap(err, "removing directory")
+	}
+	return nil
+}
+
 // frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
 func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
 
@@ -1000,6 +1034,10 @@ func (tx *RBFTx) Pointer() string {
 	return fmt.Sprintf("%p", tx)
 }
 
+func (tx *RBFTx) Dump() {
+	// todo
+}
+
 // Readonly is true if the transaction is not read-and-write, but only doing reads.
 func (tx *RBFTx) Readonly() bool {
 	return !tx.tx.Writable()
diff --git a/txfactory.go b/txfactory.go
index 99c80dd54..6120efe9f 100644
--- a/txfactory.go
+++ b/txfactory.go
@@ -21,6 +21,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"text/tabwriter"
 
 	"github.com/pilosa/pilosa/v2/rbf"
 	"github.com/pilosa/pilosa/v2/roaring"
@@ -67,25 +68,6 @@ type TxFactory struct {
 	idx *Index
 }
 
-/* want glue-green to multiplex, so don't do this directly
-// but rather f.CloseStore()
-func (f *TxFactory) Store() TxStore {
-	switch f.typeOfTx {
-	case roaringFragmentFilesTxn:
-		return &RoaringStore{}
-	case badgerTxn:
-		return f.badgerDB
-	case rbfTxn:
-		return f.rbfDB
-		//	case blueGreenBadgerRoaring:
-		//	case blueGreenRoaringBadger:
-	}
-	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
-=======
->>>>>>> Implement pilosa.Tx for RBF
-}
-*/
-
 // integer types for fast switch{}
 type txtype int
 
@@ -235,6 +217,25 @@ func (f *TxFactory) DeleteIndex(name string) error {
 	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
 }
 
+func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) error {
+	switch f.typeOfTx {
+	case roaringFragmentFilesTxn:
+		return f.roaringDB.DeleteField(index, field, fieldPath)
+	case badgerTxn:
+		return f.badgerDB.DeleteField(index, field, fieldPath)
+	case rbfTxn:
+		//return f.rbfDB.DeleteField(index, field, fieldPath)
+		return nil
+	case blueGreenBadgerRoaring:
+		_ = f.badgerDB.DeleteField(index, field, fieldPath)
+		return f.roaringDB.DeleteField(index, field, fieldPath)
+	case blueGreenRoaringBadger:
+		_ = f.roaringDB.DeleteField(index, field, fieldPath)
+		return f.badgerDB.DeleteField(index, field, fieldPath)
+	}
+	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
+}
+
 func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uint64, frag *fragment) error {
 	switch f.typeOfTx {
 	case roaringFragmentFilesTxn:
@@ -416,23 +417,33 @@ func (idx *Index) StringifiedRoaringKeys() (r string) {
 	index := idx.name
 
 	r = "allkeys:[\n"
+	n := 0
 	for _, relpath := range paths {
 		field, view, shard, err := fragmentSpecFromRoaringPath(relpath)
 		if err != nil {
 			continue // ignore .meta paths
 		}
 		abspath := idx.path + sep + relpath
-		s, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard)
+		const showOps = false
+		s, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps)
 		panicOn(err)
 		//r += fmt.Sprintf("path:'%v' fragment contains:\n") + s
+		if s == "" {
+			s = "<empty bitmap>"
+		}
 		r += s
+		n++
 	}
+	if n == 0 {
+		return "" // new convention that empty database => empty string returned.
+	}
+	// note that we can have a bitmap present, but it can be empty
 	r += "]\n   all-in-blake3:" + blake3sum16([]byte(r)) + "\n"
 
 	return "roaring-" + r
 }
 
-func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64) (r string, err error) {
+func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64, showOps bool) (r string, err error) {
 
 	var info roaring.BitmapInfo
 	_ = info
@@ -471,6 +482,21 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
 		return
 	}
 
+	//cmd.DisplayInfo(info)
+	// inlined
+	if showOps {
+		pC := pointerContext{
+			from: info.From,
+			to:   info.To,
+		}
+		if info.ContainerCount > 0 {
+			printContainers(info, pC)
+		}
+		if info.Ops > 0 {
+			printOps(info)
+		}
+	}
+
 	citer, found := rbm.Containers.Iterator(0)
 	_ = found // probably gonna use just the Ops log instead, so don't panic if !found.
 
@@ -551,28 +577,6 @@ func fileSize(name string) (int64, error) {
 
 var _ = fileSize // happy linter
 
-// Dump prints to stdout the contents of the roaring Containers
-// stored in idx. Its format may vary depending of the type of
-// idx.Txf transaction factory that is in use.
-// Mostly for debugging.
-func (idx *Index) Dump(label string) {
-	ty := idx.Txf.TxType()
-	fileline := FileLine(2)
-	switch ty {
-	case badgerTxn:
-		fmt.Printf("%v Index.Dump('%v') for index '%v':\n%v\n", fileline, label, idx.name, idx.StringifiedBadgerKeys(nil))
-		return
-	case blueGreenRoaringBadger, blueGreenBadgerRoaring:
-		fmt.Printf("%v Index.Dump('%v') for index '%v', RoaringTx:\n%v\n", fileline, label, idx.name, idx.StringifiedRoaringKeys())
-		fmt.Printf("%v Index.Dump('%v') for index '%v', BadgerTx :\n%v\n", fileline, label, idx.name, idx.StringifiedBadgerKeys(nil))
-		return
-	case roaringFragmentFilesTxn:
-		fmt.Printf("%v Index.Dump('%v') for index '%v', BadgerTx :\n%v\n", fileline, label, idx.name, idx.StringifiedRoaringKeys())
-		return
-	}
-	panic(fmt.Errorf("%v Index.Dump('%v') for index '%v': no implementation for txtype '%v'\n", fileline, label, idx.name, ty))
-}
-
 func containerToBytes(ct *roaring.Container) []byte {
 	ty := roaring.ContainerType(ct)
 	switch ty {
@@ -587,3 +591,109 @@ func containerToBytes(ct *roaring.Container) []byte {
 	}
 	panic(fmt.Sprintf("unknown container type '%v'", int(ty)))
 }
+
+type pointerContext struct {
+	from, to uintptr
+}
+
+func printOps(info roaring.BitmapInfo) {
+	fmt.Fprintln(os.Stdout, "  Ops:")
+	tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
+	printed := 0
+	for _, op := range info.OpDetails {
+		fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
+		printed++
+	}
+	tw.Flush()
+}
+
+func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
+	var pointer string
+	if c.Mapped {
+		if c.Pointer >= p.from && c.Pointer < p.to {
+			pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
+		} else {
+			pointer = fmt.Sprintf("!0x%x!", c.Pointer)
+		}
+	} else {
+		pointer = fmt.Sprintf("0x%x", c.Pointer)
+	}
+	return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
+}
+
+// stolen from ctl/inspect.go
+func printContainers(info roaring.BitmapInfo, pC pointerContext) {
+	fmt.Fprintln(os.Stdout, "  Containers:")
+	tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
+	fmt.Fprintf(tw, "  \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
+	fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
+	c1s := info.Containers
+	c2s := info.OpContainers
+	l1 := len(c1s)
+	l2 := len(c2s)
+	i1 := 0
+	i2 := 0
+	var c1, c2 roaring.ContainerInfo
+	c1.Key = ^uint64(0)
+	c2.Key = ^uint64(0)
+	c1e := false
+	c2e := false
+	if i1 < l1 {
+		c1 = c1s[i1]
+		i1++
+		c1e = true
+	}
+	if i2 < l2 {
+		c2 = c2s[i2]
+		i2++
+		c2e = true
+	}
+	printed := 0
+	for c1e || c2e {
+		c1used := false
+		c2used := false
+		var key uint64
+		c1fmt := "-\t\t\t"
+		c2fmt := "-\t\t\t"
+		// If c2 exists, we'll always prefer its flags,
+		// if it doesn't, this gets overwritten.
+		flags := c2.Flags
+		if !c2e || (c1e && c1.Key < c2.Key) {
+			c1fmt = pC.pretty(c1)
+			key = c1.Key
+			c1used = true
+			flags = c1.Flags
+		} else if !c1e || (c2e && c2.Key < c1.Key) {
+			c2fmt = pC.pretty(c2)
+			key = c2.Key
+			c2used = true
+		} else {
+			// c1e and c2e both set, and neither key is < the other.
+			c1fmt = pC.pretty(c1)
+			c2fmt = pC.pretty(c2)
+			key = c1.Key
+			c1used = true
+			c2used = true
+		}
+		if c1used {
+			if i1 < l1 {
+				c1 = c1s[i1]
+				i1++
+			} else {
+				c1e = false
+			}
+		}
+		if c2used {
+			if i2 < l2 {
+				c2 = c2s[i2]
+				i2++
+			} else {
+				c2e = false
+			}
+		}
+		fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
+		printed++
+	}
+	tw.Flush()
+}
diff --git a/utils_internal_test.go b/utils_internal_test.go
index d87499a88..f24fded67 100644
--- a/utils_internal_test.go
+++ b/utils_internal_test.go
@@ -458,6 +458,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
 			srctx := srcIdx.Txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment})
 
 			destIdx := destCluster.holder.Index(src.Index)
+
 			desttx := destIdx.Txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment})
 
 			citer, _, err := srctx.ContainerIterator(src.Index, src.Field, src.View, src.Shard, 0)
diff --git a/view.go b/view.go
index da61f1837..430845fbb 100644
--- a/view.go
+++ b/view.go
@@ -137,9 +137,12 @@ func (v *view) open() error {
 	if err := func() error {
 		// Ensure the view's path exists.
 		v.holder.Logger.Debugf("ensure view path exists: %s", v.path)
-		if err := os.MkdirAll(v.path, 0777); err != nil {
+		err := os.MkdirAll(v.path, 0777)
+		if err != nil {
 			return errors.Wrap(err, "creating view directory")
-		} else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil {
+		}
+		err = os.MkdirAll(filepath.Join(v.path, "fragments"), 0777)
+		if err != nil {
 			return errors.Wrap(err, "creating fragments directory")
 		}
 
diff --git a/vprint.go b/vprint.go
index 52130bbeb..05159615b 100644
--- a/vprint.go
+++ b/vprint.go
@@ -141,3 +141,27 @@ func FileSize(name string) (int64, error) {
 	}
 	return fi.Size(), nil
 }
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}

From 8903d8c117be422e404a13c10de52885f8a63993 Mon Sep 17 00:00:00 2001
From: Jason Aten <jason.aten@molecula.com>
Date: Thu, 30 Jul 2020 12:35:49 -0400
Subject: [PATCH 12/14] rbf Dump() and DumpString() debug methods.

---
 Makefile                  |   6 --
 fragment_internal_test.go |   3 +
 license.exceptions        |   1 +
 rbf/blake3.go             |  97 ++++++++++++++++++++++
 rbf/tx.go                 | 164 +++++++++++++++++++++++++++++++++++-
 rbf/tx_test.go            |  26 ++++++
 rbf/vprint.go             | 169 ++++++++++++++++++++++++++++++++++++++
 tx.go                     |   5 +-
 8 files changed, 462 insertions(+), 9 deletions(-)
 create mode 100644 rbf/blake3.go
 create mode 100644 rbf/vprint.go

diff --git a/Makefile b/Makefile
index e794990ee..e38ed3860 100644
--- a/Makefile
+++ b/Makefile
@@ -160,12 +160,6 @@ topt-badger:
 	@echo "   log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l
 	@echo "   log.topt.badger   red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
 
-topt-rb:
-	mv log.topt.roaring_badger log.topt.roaring_badger.prev || true
-	PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger
-	@echo "   log.topt.roaring_badger green: \c"; cat log.topt.roaring_badger | grep PASS |wc -l
-	@echo "   log.topt.roaring_badger   red: \c"; cat log.topt.roaring_badger | grep '\-\-\- FAIL' |wc -l
-
 topt-badger-race:
 	mv log.topt.badger-race log.topt.badger-race.prev || true
 	PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)  2>&1 | tee log.topt.badger-race
diff --git a/fragment_internal_test.go b/fragment_internal_test.go
index f30e27ef2..a8851a1bb 100644
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@@ -3902,6 +3902,7 @@ func TestFragment_RoaringImport(t *testing.T) {
 			defer tx.Rollback()
 
 			for num, input := range test {
+				vv("num=%v, input='%#v'", num, input)
 				buf := &bytes.Buffer{}
 				bm := roaring.NewBitmap(input...)
 				_, err := bm.WriteTo(buf)
@@ -3912,6 +3913,8 @@ func TestFragment_RoaringImport(t *testing.T) {
 				if err != nil {
 					t.Fatalf("importing roaring: %v", err)
 				}
+				tx.Dump()
+
 				exp := calcExpected(test[:num+1]...)
 				for row, expCols := range exp {
 					cols := f.mustRow(tx, uint64(row)).Columns()
diff --git a/license.exceptions b/license.exceptions
index e703b7c2a..67767e1e2 100644
--- a/license.exceptions
+++ b/license.exceptions
@@ -10,3 +10,4 @@
 ./logger/filewriter.go
 ./logger/filewriter_test.go
 ./vprint.go
+./rbf/vprint.go
diff --git a/rbf/blake3.go b/rbf/blake3.go
new file mode 100644
index 000000000..3935d2576
--- /dev/null
+++ b/rbf/blake3.go
@@ -0,0 +1,97 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rbf
+
+import (
+	"encoding/binary"
+	"fmt"
+	"sync"
+
+	cryptorand "crypto/rand"
+	"github.com/zeebo/blake3"
+)
+
+// Blake3Hasher is a thread/goroutine safe way to
+// obtain a blake3 cryptographic hash of input []byte.
+// Reference https://github.com/BLAKE3-team/BLAKE3
+// suggests it is 6x faster than BLAKE2B.
+// The Go github.com/zeebo/blake3 version is
+// AVX2 and SSE4.1 accelerated.
+type Blake3Hasher struct {
+	hasher   *blake3.Hasher
+	hasherMu sync.Mutex
+}
+
+// NewBlake3Hasher returns a new Blake3Hasher.
+func NewBlake3Hasher() *Blake3Hasher {
+	return &Blake3Hasher{
+		hasher: blake3.New(),
+	}
+}
+
+// CryptoHash writes the blake3 cryptographic hash of
+// input into buffer and returns it.
+// Like the standard libary's hash.Hash interface's Sum() method,
+// the buffer is re-used and overwritten
+// to avoid allocation. The caller determines the byte length of
+// the outputCryptohash by the size of the supplied buffer
+// slice, and this will be exactly equal to the supplies bytes.
+// In this way, shorter or longer hashes can be provided as
+// needed.
+func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash []byte) {
+	w.hasherMu.Lock()
+	w.hasher.Reset()
+
+	// "Write implements part of the hash.Hash interface. It never returns an error."
+	//  -- https://godoc.org/github.com/zeebo/blake3#Hasher.Write
+	_, _ = w.hasher.Write(input)
+
+	// Digest.Read reads data from the hasher into buffer.
+	// "It always fills the entire buffer and never errors."
+	//   -- https://godoc.org/github.com/zeebo/blake3#Digest
+	_, _ = w.hasher.Digest().Read(buffer)
+
+	// no chance of panic, so avoid any defer cost.
+	w.hasherMu.Unlock()
+
+	return buffer
+}
+
+// blake3sum16 might be slower because we allocate a new hasher every time, but
+// it is more conenient for writing debug code. It returns
+// a 16 byte hash as a hexidecimal string.
+func blake3sum16(input []byte) string {
+	hasher := blake3.New()
+
+	_, _ = hasher.Write(input)
+	var buf [16]byte
+	_, _ = hasher.Digest().Read(buf[0:])
+
+	return fmt.Sprintf("%x", buf)
+}
+
+// cryptoRandInt64 uses crypto/rand to get an random int64
+func cryptoRandInt64() int64 {
+	c := 8
+	b := make([]byte, c)
+	_, err := cryptorand.Read(b)
+	if err != nil {
+		panic(err)
+	}
+	r := int64(binary.LittleEndian.Uint64(b))
+	return r
+}
+
+var _ = cryptoRandInt64 // happy linter
diff --git a/rbf/tx.go b/rbf/tx.go
index 83f1ae244..ca208fd1b 100644
--- a/rbf/tx.go
+++ b/rbf/tx.go
@@ -14,10 +14,12 @@
 package rbf
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"math"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -1152,7 +1154,7 @@ func (itr *containerIterator) Close() {}
 // Next moves the iterator to the next container.
 func (itr *containerIterator) Next() bool {
 	err := itr.cursor.Next()
-	return err != nil
+	return err == nil
 }
 
 // Value returns the current key & container.
@@ -1160,3 +1162,163 @@ func (itr *containerIterator) Value() (uint64, *roaring.Container) {
 	cell := itr.cursor.cell()
 	return cell.Key, toContainer(cell, itr.cursor.tx)
 }
+
+func (tx *Tx) Dump(index string) {
+	fmt.Println(tx.DumpString(index))
+}
+func (tx *Tx) DumpString(index string) (r string) {
+
+	r = "allkeys:[\n"
+
+	// grab root records, for a list of bitmaps.
+	records, err := tx.rootRecords()
+	panicOn(err)
+	n := 0
+	for _, rr := range records {
+		c, err := tx.cursor(rr.Name)
+		panicOn(err)
+		err = c.First()
+		if err == io.EOF {
+			r += "<empty bitmap>"
+			n++
+			continue
+		}
+		panicOn(err)
+		for {
+			err := c.Next() // hung here?
+			if err == io.EOF {
+				break
+			}
+			panicOn(err)
+			cell := c.cell()
+			ckey := cell.Key
+			ct := toContainer(cell, tx)
+
+			s := stringOfCkeyCt(ckey, ct, rr.Name, index)
+			r += s
+			n++
+		}
+	}
+	if n == 0 {
+		return ""
+	}
+	// note that we can have a bitmap present, but it can be empty
+	r += "]\n   all-in-blake3:" + blake3sum16([]byte(r)) + "\n"
+
+	return "rbf-" + r
+}
+
+func containerToBytes(ct *roaring.Container) []byte {
+	ty := roaring.ContainerType(ct)
+	switch ty {
+	case containerNil:
+		panic("nil container")
+	case containerArray:
+		return fromArray16(roaring.AsArray(ct))
+	case containerBitmap:
+		return fromArray64(roaring.AsBitmap(ct))
+	case containerRun:
+		return fromInterval16(roaring.AsRuns(ct))
+	}
+	panic(fmt.Sprintf("unknown container type '%v'", int(ty)))
+}
+
+func badgerKey(index, field, view string, shard uint64, roaringContainerKey uint64) []byte {
+	// The %020d which adds zero padding up to 20 runes is required to
+	// allow the textual sort to accurately
+	// reflect a numeric sort order. This is because, as a string,
+	// math.MaxUint64 is 20 bytes long.
+	// Example of such a badgerKey with a container-key that is math.MaxUint64:
+	// ...........................................12345678901234567890
+	// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615
+
+	prefix := badgerPrefix(index, field, view, shard)
+	ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
+	bkey := append(prefix, ckey...)
+	MustValidateKey(bkey)
+	return bkey
+}
+
+// badgerPrefix returns everything from badgerKey up to and
+// including the '@' fune in a badger key. The prefix excludes the roaring container key itself.
+// NB must be kept in sync with badgerKey() and badgerKeyExtractContainerKey().
+func badgerPrefix(index, field, view string, shard uint64) []byte {
+	return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard))
+}
+
+// MustValidatekey will panic on a bad badgerKey with an informative message.
+func MustValidateKey(bkey []byte) {
+	n := len(bkey)
+	if n < 56 {
+		panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey)))
+	}
+	beforeCkey := bkey[n-26 : n-20]
+	if !bytes.Equal(beforeCkey, ckeyPartExpected) {
+		panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey)))
+	}
+}
+
+func bitmapAsString(rbm *roaring.Bitmap) (r string) {
+	r = "c("
+	slc := rbm.Slice()
+	width := 0
+	s := ""
+	for _, v := range slc {
+		if width == 0 {
+			s = fmt.Sprintf("%v", v)
+		} else {
+			s = fmt.Sprintf(", %v", v)
+		}
+		width += len(s)
+		r += s
+		if width > 70 {
+			r += ",\n"
+			width = 0
+		}
+	}
+	if width == 0 && len(r) > 2 {
+		r = r[:len(r)-2]
+	}
+	return r + ")"
+}
+
+// should really be exported from the pilosa/roaring package so we don't get out of sync...
+const (
+	containerNil    byte = iota // no container
+	containerArray              // slice of bit position values
+	containerBitmap             // slice of 1024 uint64s
+	containerRun                // container of run-encoded bits
+)
+
+var ckeyPartExpected = []byte(";ckey@")
+
+func invName(rbfName string) (field, view string, shard uint64) {
+	s := strings.Split(rbfName, "\x00")
+	if len(s) != 3 {
+		panic("should have 3 parts")
+	}
+	field = s[0]
+	view = s[1]
+	var err error
+	shard, err = strconv.ParseUint(s[2], 10, 64)
+	panicOn(err)
+	return
+}
+
+func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName, index string) (s string) {
+
+	by := containerToBytes(ct)
+	hash := blake3sum16(by)
+
+	cts := roaring.NewSliceContainers()
+	cts.Put(ckey, ct)
+	rbm := &roaring.Bitmap{Containers: cts}
+	srbm := bitmapAsString(rbm)
+
+	field, view, shard := invName(rrName)
+	bkey := string(badgerKey(index, field, view, shard, ckey))
+
+	s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N())
+	s += "          ......." + srbm + "\n"
+	return
+}
diff --git a/rbf/tx_test.go b/rbf/tx_test.go
index 85195ad1d..e245b73da 100644
--- a/rbf/tx_test.go
+++ b/rbf/tx_test.go
@@ -464,3 +464,29 @@ func BenchmarkTx_Contains(b *testing.B) {
 		})
 	}
 }
+
+func TestTx_Dump(t *testing.T) {
+	db := MustOpenDB(t)
+	defer MustCloseDB(t, db)
+	tx := MustBegin(t, db, true)
+	defer tx.Rollback()
+
+	index, field, view, shard := "i", "f", "v", uint64(15)
+	nm := rbfName(field, view, shard)
+
+	if err := tx.CreateBitmap(nm); err != nil {
+		t.Fatal(err)
+	} else if _, err := tx.Add(nm, 0x00000001, 0x00000002, 0x00010003, 0x00030004); err != nil {
+		t.Fatal(err)
+	}
+
+	// test that we don't crash, and get *something* back
+	s := tx.DumpString(index)
+	if s == "" {
+		panic("should have had 3 containers!")
+	}
+}
+
+func rbfName(field, view string, shard uint64) string {
+	return fmt.Sprintf("%s\x00%s\x00%d", field, view, shard)
+}
diff --git a/rbf/vprint.go b/rbf/vprint.go
new file mode 100644
index 000000000..dfd630c70
--- /dev/null
+++ b/rbf/vprint.go
@@ -0,0 +1,169 @@
+// home: https://github.com/glyerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package rbf
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+var _ = stack // happy linter
diff --git a/tx.go b/tx.go
index 051397245..ff544aca7 100644
--- a/tx.go
+++ b/tx.go
@@ -910,7 +910,8 @@ func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64
 }
 
 type RBFTx struct {
-	tx *rbf.Tx
+	index string
+	tx    *rbf.Tx
 }
 
 func (tx *RBFTx) Type() string {
@@ -1035,7 +1036,7 @@ func (tx *RBFTx) Pointer() string {
 }
 
 func (tx *RBFTx) Dump() {
-	// todo
+	tx.tx.Dump(tx.index)
 }
 
 // Readonly is true if the transaction is not read-and-write, but only doing reads.

From a3d802f8a3ed500df58fc9776cc8245d0b0519eb Mon Sep 17 00:00:00 2001
From: Jason Aten <jason.aten@molecula.com>
Date: Thu, 30 Jul 2020 14:06:42 -0400
Subject: [PATCH 13/14] rbf: OffsetRange, ImportRoaringBits, CountRange work

green:
TestFragment_RowsIteration/combinations
TestFragment_RoaringImportTopN

red: (needs Ben's attention)
PILOSA_TXSRC=rbf go test -v -run TestFragment_TopN_IDs  -tags=' shardwidth20'  "-gcflags=all=-d=checkptr=0"

also red: (one for Ben)
TestCursor_FirstNext_Quick/9 is throwing
  panic: cannot find segment containing WAL page: 1
  as we check the error back from checkpoint() in Rollback().
---
 badger.go                 |   5 +
 bluegreentx.go            |   2 +-
 executor_test.go          |   4 +-
 fragment_internal_test.go |   3 +-
 index.go                  |  17 +-
 like_test.go              |   1 -
 rbf/rbf.go                |  27 +++-
 rbf/tx.go                 | 320 +++++++++++++++++++++++++++++++++-----
 tx.go                     |   3 +-
 txfactory.go              |  91 ++++++-----
 10 files changed, 380 insertions(+), 93 deletions(-)

diff --git a/badger.go b/badger.go
index f2fca4dc6..ce28de54e 100644
--- a/badger.go
+++ b/badger.go
@@ -1317,6 +1317,10 @@ func (tx *BadgerTx) UnionInPlace(index, field, view string, shard uint64, others
 // roaring.countRange counts the number of bits set between [start, end).
 func (tx *BadgerTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
 
+	if start >= end {
+		return 0, nil
+	}
+
 	skey := highbits(start)
 	ekey := highbits(end)
 
@@ -1415,6 +1419,7 @@ func (tx *BadgerTx) OffsetRange(index, field, view string, shard, offset, start,
 		bkey := item.Key()
 		k := badgerKeyExtractContainerKey(bkey)
 
+		// >= hi1 is correct b/c endx cannot have any lowbits set.
 		if uint64(k) >= hi1 {
 			break
 		}
diff --git a/bluegreentx.go b/bluegreentx.go
index 9df2596a1..3084a2390 100644
--- a/bluegreentx.go
+++ b/bluegreentx.go
@@ -130,7 +130,6 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
 			panic(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, stack()))
 		}
 	}
-
 	for aIter.Next() {
 		aKey, aValue := aIter.Value()
 
@@ -582,6 +581,7 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe
 
 func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
 	c.checker.see(index, field, view, shard)
+	//vv("CountRange start=0x%x, endx=0x%x", start, end)
 	defer func() {
 		if r := recover(); r != nil {
 			c.Dump()
diff --git a/executor_test.go b/executor_test.go
index dd4927209..6fd5a7b61 100644
--- a/executor_test.go
+++ b/executor_test.go
@@ -4096,8 +4096,8 @@ func TestExecutor_Execute_SetRow(t *testing.T) {
 		c := test.MustRunCluster(t, 1)
 		defer c.Close()
 		hldr := test.Holder{Holder: c[0].Server.Holder()}
-		index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
-		_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
+		idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
+		_, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
 		if err != nil {
 			t.Fatal(err)
 		}
diff --git a/fragment_internal_test.go b/fragment_internal_test.go
index a8851a1bb..871864d3d 100644
--- a/fragment_internal_test.go
+++ b/fragment_internal_test.go
@@ -3902,7 +3902,6 @@ func TestFragment_RoaringImport(t *testing.T) {
 			defer tx.Rollback()
 
 			for num, input := range test {
-				vv("num=%v, input='%#v'", num, input)
 				buf := &bytes.Buffer{}
 				bm := roaring.NewBitmap(input...)
 				_, err := bm.WriteTo(buf)
@@ -3913,7 +3912,6 @@ func TestFragment_RoaringImport(t *testing.T) {
 				if err != nil {
 					t.Fatalf("importing roaring: %v", err)
 				}
-				tx.Dump()
 
 				exp := calcExpected(test[:num+1]...)
 				for row, expCols := range exp {
@@ -3960,6 +3958,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) {
 			if err != nil {
 				t.Fatalf("bulk importing ids: %v", err)
 			}
+
 			expPairs := calcTop(test.rowIDs, test.colIDs)
 			pairs, err := f.top(tx, topOptions{})
 			if err != nil {
diff --git a/index.go b/index.go
index 6007af4f8..d1a7770df 100644
--- a/index.go
+++ b/index.go
@@ -72,8 +72,21 @@ type Index struct {
 	Txf *TxFactory
 }
 
-// NewIndex returns a new instance of Index.
+// OpenIndex opens or starts a new Index on path. Path
+// can be empty.
+func OpenIndex(holder *Holder, path, name string) (*Index, error) {
+	openExisting := true
+	return openOrCreateNewIndex(holder, path, name, openExisting)
+}
+
+// NewIndex returns a new instance of Index at path. It will erase anything
+// old already in path.
 func NewIndex(holder *Holder, path, name string) (*Index, error) {
+	openExisting := false
+	return openOrCreateNewIndex(holder, path, name, openExisting)
+}
+
+func openOrCreateNewIndex(holder *Holder, path, name string, openExisting bool) (*Index, error) {
 
 	// Emulate what the spf13/cobra does, letting env vars override
 	// the defaults, because we may be under a simple "go test" run where
@@ -104,7 +117,7 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
 		return nil, errors.Wrap(err, "validating name")
 	}
 
-	txf, err := NewTxFactory(txsrc, holder.Path, name)
+	txf, err := NewTxFactory(txsrc, holder.Path, name, openExisting)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating newTxFactory")
 	}
diff --git a/like_test.go b/like_test.go
index 1382901ec..7e5268dea 100644
--- a/like_test.go
+++ b/like_test.go
@@ -20,7 +20,6 @@ import (
 )
 
 func TestPlanLike(t *testing.T) {
-	t.Parallel()
 
 	cases := []struct {
 		name            string
diff --git a/rbf/rbf.go b/rbf/rbf.go
index 2aa3a473a..911532d15 100644
--- a/rbf/rbf.go
+++ b/rbf/rbf.go
@@ -262,8 +262,14 @@ func align8(offset int) int {
 // leafCell represents a leaf cell.
 type leafCell struct {
 	Key  uint64
-	Type int
-	N    int
+	Type int // container type
+
+	// N is the number of "things" in Data:
+	//  for an array container the number of integers in the array.
+	//  for an RLE, number of intervals.
+	// etc.
+	N int
+
 	BitN int
 	Data []byte
 }
@@ -392,19 +398,22 @@ func (c *leafCell) lastValue() uint16 {
 }
 
 // countRange returns the bit count within the given range.
-func (c *leafCell) countRange(start, end uint16) (n int) {
+// We have to take int32 rather than uint16 because the interval is [start, end),
+// and otherwise we have no way to ask to count the entire container (the
+// high bit will be missed).
+func (c *leafCell) countRange(start, end int32) (n int) {
 	// If the full range is being queried, simply use the precalculated count.
-	if start == 0 && end == math.MaxUint16 {
+	if start == 0 && end > math.MaxUint16 {
 		return c.BitN
 	}
 
 	switch c.Type {
 	case ContainerTypeArray:
-		return int(roaring.ArrayCountRange(toArray16(c.Data), int32(start), int32(end)))
+		return int(roaring.ArrayCountRange(toArray16(c.Data), start, end))
 	case ContainerTypeRLE:
-		return int(roaring.RunCountRange(toInterval16(c.Data), int32(start), int32(end)))
+		return int(roaring.RunCountRange(toInterval16(c.Data), start, end))
 	case ContainerTypeBitmap:
-		return int(roaring.BitmapCountRange(toArray64(c.Data), int32(start), int32(end)))
+		return int(roaring.BitmapCountRange(toArray64(c.Data), start, end))
 	default:
 		panic(fmt.Sprintf("invalid container type: %d", c.Type))
 	}
@@ -412,11 +421,15 @@ func (c *leafCell) countRange(start, end uint16) (n int) {
 
 func readLeafCellKey(page []byte, i int) uint64 {
 	offset := readCellOffset(page, i)
+	assert(offset < len(page))
 	return *(*uint64)(unsafe.Pointer(&page[offset]))
 }
 
 func readLeafCell(page []byte, i int) leafCell {
 	offset := readCellOffset(page, i)
+
+	// cd ..; PILOSA_TXSRC=rbf go test -v -run TestFragment_TopN_IDs  -tags=' shardwidth20'  "-gcflags=all=-d=checkptr=0"
+	// gives panic: runtime error: slice bounds out of range [16390:8192] here.
 	buf := page[offset:]
 
 	var cell leafCell
diff --git a/rbf/tx.go b/rbf/tx.go
index ca208fd1b..d11c34909 100644
--- a/rbf/tx.go
+++ b/rbf/tx.go
@@ -36,6 +36,15 @@ type Tx struct {
 	pageMap  *immutable.Map // mapping of database pages to WAL IDs
 	writable bool           // if true, tx can write
 	dirty    bool           // if true, changes have been made
+
+	// If Rollback() has already completed, don't do it again.
+	// Note db == nil means that commit has already been done.
+	rollbackDone bool
+
+	// DeleteEmptyContainer lets us by default match the roaring
+	// behavior where an existing container has all its bits cleared
+	// but still sticks around in the database.
+	DeleteEmptyContainer bool
 }
 
 // Writable returns true if the transaction can mutate data.
@@ -74,12 +83,17 @@ func (tx *Tx) Commit() error {
 func (tx *Tx) Rollback() {
 	tx.mu.Lock()
 	defer tx.mu.Unlock()
-
-	// TODO(bbj): Invalidate DB if rollback fails. Possibly attempt reopen?
-
-	if tx.db == nil {
+	// allow Rollback to be called more than once.
+	if tx.rollbackDone {
 		return
 	}
+	tx.rollbackDone = true
+	if tx.db == nil {
+		// Commit already done.
+		return
+	}
+
+	// TODO(bbj): Invalidate DB if rollback fails. Possibly attempt reopen?
 
 	// If any pages have been written, ensure we write a new meta page with
 	// the rollback flag to mark the end of the transaction. This allows us to
@@ -92,10 +106,18 @@ func (tx *Tx) Rollback() {
 		}
 	}
 
-	_ = tx.db.checkpoint() // TODO: Check error
+	// turn on these error checks! we see
+	// panic: cannot find segment containing WAL page: 1
+	// when running go test -v
+	// TestCursor_FirstNext_Quick/6
+	//
+	//panicOn(tx.db.checkpoint())
+	//panicOn(tx.db.removeTx(tx))
+
+	_ = tx.db.checkpoint()
 
 	// Disconnect transaction from DB.
-	_ = tx.db.removeTx(tx) // TODO: Check error
+	_ = tx.db.removeTx(tx)
 }
 
 // Root returns the root page number for a bitmap. Returns 0 if the bitmap does not exist.
@@ -150,6 +172,8 @@ func (tx *Tx) CreateBitmap(name string) error {
 }
 
 func (tx *Tx) createBitmap(name string) error {
+	//vv("createBitmap(name='%v'", name)
+
 	if tx.db == nil {
 		return ErrTxClosed
 	} else if !tx.writable {
@@ -425,6 +449,8 @@ func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) {
 
 // Add sets a given bit on the bitmap.
 func (tx *Tx) Add(name string, a ...uint64) (changeCount int, err error) {
+	//vv("rbf Tx.Add(a='%#v')", a)
+
 	tx.mu.Lock()
 	defer tx.mu.Unlock()
 
@@ -589,14 +615,14 @@ func (tx *Tx) Container(name string, key uint64) (*roaring.Container, error) {
 }
 
 // PutContainer inserts a container into a bitmap. Overwrites if key already exists.
-func (tx *Tx) PutContainer(name string, key uint64, cont *roaring.Container) error {
+func (tx *Tx) PutContainer(name string, key uint64, ct *roaring.Container) error {
 	tx.mu.Lock()
 	defer tx.mu.Unlock()
 
-	cell := ConvertToLeafArgs(key, cont)
-	if cell.BitN == 0 {
+	if ct.N() == 0 {
 		return nil
 	}
+	cell := ConvertToLeafArgs(key, ct)
 
 	if err := tx.createBitmapIfNotExists(name); err != nil {
 		return err
@@ -939,16 +965,27 @@ func (tx *Tx) ContainerIterator(name string, key uint64) (citer roaring.Containe
 	tx.mu.RLock()
 	defer tx.mu.RUnlock()
 
-	c, err := tx.cursor(name)
-	if err != nil {
-		// TODO(bbj): Don't return error if bitmap is simply not found?
-		return nil, false, err
-	} else if c == nil {
-		return nil, false, nil
-	} else if err := c.First(); err != nil {
-		return nil, false, err
+	var c *Cursor
+	c, err = tx.cursor(name)
+	if c == nil && err == nil {
+		// nothing available.
+		citer = &emptyContainerIterator{}
+		return
 	}
-	return &containerIterator{cursor: c}, true, nil
+	if err != nil {
+		return
+	}
+
+	// INVAR: c is not nil
+
+	err = c.First()
+	if err != nil {
+		return
+	}
+	ci := &containerIterator{cursor: c}
+	citer = ci
+
+	return citer, true, nil
 }
 
 func (tx *Tx) ForEach(name string, fn func(i uint64) error) error {
@@ -1091,18 +1128,28 @@ func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error {
 	panic("TODO")
 }
 
+// roaring.countRange counts the number of bits set between [start, end).
 func (tx *Tx) CountRange(name string, start, end uint64) (uint64, error) {
 	tx.mu.RLock()
 	defer tx.mu.RUnlock()
 
-	c, err := tx.cursor(name)
-	if err != nil {
-		return 0, err
-	} else if c == nil {
+	if start >= end {
 		return 0, nil
 	}
 
-	if err := c.First(); err == io.EOF {
+	skey := highbits(start)
+	ekey := highbits(end)
+
+	csr, err := tx.cursor(name)
+	if err != nil {
+		return 0, err
+	} else if csr == nil {
+		return 0, nil
+	}
+
+	exact, err := csr.Seek(skey)
+	_ = exact
+	if err == io.EOF {
 		return 0, nil
 	} else if err != nil {
 		return 0, err
@@ -1110,37 +1157,94 @@ func (tx *Tx) CountRange(name string, start, end uint64) (uint64, error) {
 
 	var n uint64
 	for {
-		if err := c.Next(); err == io.EOF {
+		if err := csr.Next(); err == io.EOF {
 			break
 		} else if err != nil {
 			return 0, err
 		}
 
-		cell := c.cell()
-		if cell.Key > highbits(end) {
+		c := csr.cell()
+		k := c.Key
+		if k > ekey {
 			break
 		}
 
-		if cell.Key == highbits(start) {
-			n += uint64(cell.countRange(lowbits(start), math.MaxUint16))
-		} else if cell.Key == highbits(end) {
-			n += uint64(cell.countRange(0, lowbits(end)))
-		} else {
-			n += uint64(cell.BitN)
+		// If range is entirely in one container then just count that range.
+		if skey == ekey {
+			return uint64(c.countRange(int32(lowbits(start)), int32(lowbits(end)))), nil
+		}
+		// INVAR: skey < ekey
+
+		// k > ekey handles the case when start > end and where start and end
+		// are in different containers. Same container case is already handled above.
+		if k > ekey {
+			break
+		}
+		if k == skey {
+			n += uint64(c.countRange(int32(lowbits(start)), roaring.MaxContainerVal+1))
+			continue
+		}
+		if k < ekey {
+			n += uint64(c.BitN)
+			continue
+		}
+		if k == ekey {
+			n += uint64(c.countRange(0, int32(lowbits(end))))
+			break
 		}
 	}
 	return n, nil
 }
 
-func (tx *Tx) OffsetRange(name string, offset, start, end uint64) (*roaring.Bitmap, error) {
+func (tx *Tx) OffsetRange(name string, offset, start, endx uint64) (*roaring.Bitmap, error) {
+	if lowbits(offset) != 0 {
+		panic("offset must not contain low bits")
+	} else if lowbits(start) != 0 {
+		panic("range start must not contain low bits")
+	} else if lowbits(endx) != 0 {
+		panic("range endx must not contain low bits")
+	}
+
 	tx.mu.RLock()
 	defer tx.mu.RUnlock()
 
-	b, err := tx.RoaringBitmap(name)
+	c, err := tx.cursor(name)
 	if err != nil {
 		return nil, err
 	}
-	return b.OffsetRange(offset, start, end), nil
+
+	other := roaring.NewSliceBitmap()
+	off := highbits(offset)
+	hi0, hi1 := highbits(start), highbits(endx)
+
+	if c == nil {
+		// bitmap not found. Match what roaring does and return nil in this case.
+		return other, nil
+	}
+
+	if _, err := c.Seek(hi0); err == io.EOF {
+		return other, nil
+	} else if err != nil {
+		return nil, err
+	}
+
+	for {
+		if err := c.Next(); err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		cell := c.cell()
+		ckey := cell.Key
+
+		// >= hi1 is correct b/c endx cannot have any lowbits set.
+		if ckey >= hi1 {
+			break
+		}
+		other.Containers.Put(off+(ckey-hi0), toContainer(cell, tx))
+	}
+	return other, nil
 }
 
 // containerIterator wraps Cursor to implement roaring.ContainerIterator.
@@ -1163,6 +1267,18 @@ func (itr *containerIterator) Value() (uint64, *roaring.Container) {
 	return cell.Key, toContainer(cell, itr.cursor.tx)
 }
 
+// always returns false for Next()
+type emptyContainerIterator struct{}
+
+func (si *emptyContainerIterator) Close() {}
+
+func (si *emptyContainerIterator) Next() bool {
+	return false
+}
+func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) {
+	panic("emptyContainerIterator never has any Values")
+}
+
 func (tx *Tx) Dump(index string) {
 	fmt.Println(tx.DumpString(index))
 }
@@ -1177,7 +1293,7 @@ func (tx *Tx) DumpString(index string) (r string) {
 	for _, rr := range records {
 		c, err := tx.cursor(rr.Name)
 		panicOn(err)
-		err = c.First()
+		err = c.First() // First will rewind to beginning.
 		if err == io.EOF {
 			r += "<empty bitmap>"
 			n++
@@ -1185,7 +1301,7 @@ func (tx *Tx) DumpString(index string) (r string) {
 		}
 		panicOn(err)
 		for {
-			err := c.Next() // hung here?
+			err := c.Next()
 			if err == io.EOF {
 				break
 			}
@@ -1322,3 +1438,133 @@ func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName, index string) (s
 	s += "          ......." + srbm + "\n"
 	return
 }
+
+func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
+
+	// begin write boilerplate
+	if tx.db == nil {
+		err = ErrTxClosed
+		return
+	} else if !tx.writable {
+		err = ErrTxNotWritable
+		return
+	} else if name == "" {
+		err = ErrBitmapNameRequired
+		return
+	}
+
+	if err = tx.createBitmapIfNotExists(name); err != nil {
+		return
+	}
+	// end write boilerplate
+
+	n := itr.Len()
+	if n == 0 {
+		return
+	}
+	rowSet = make(map[uint64]int)
+
+	var currRow uint64
+
+	var oldC *roaring.Container
+	for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() {
+		if rowSize != 0 {
+			currRow = itrKey / rowSize
+		}
+		nsynth := int(synthC.N())
+		if nsynth == 0 {
+			continue
+		}
+		// INVAR: nsynth > 0
+
+		oldC, err = tx.Container(name, itrKey)
+		panicOn(err)
+		if err != nil {
+			return
+		}
+
+		if oldC == nil || oldC.N() == 0 {
+			// no container at the itrKey in badger (or all zero container).
+			if clear {
+				// changed of 0 and empty rowSet is perfect, no need to change the defaults.
+				continue
+			} else {
+
+				changed += nsynth
+				rowSet[currRow] += nsynth
+
+				err = tx.PutContainer(name, itrKey, synthC)
+				if err != nil {
+					return
+				}
+				continue
+			}
+		}
+
+		if clear {
+			existN := oldC.N() // number of bits set in the old container
+			newC := oldC.Difference(synthC)
+
+			// update rowSet and changes
+			if newC.N() == existN {
+				// INVAR: do changed need adjusting? nope. same bit count,
+				// so no change could have happened.
+				continue
+			} else {
+				changes := int(existN - newC.N())
+				changed += changes
+				rowSet[currRow] -= changes
+
+				if tx.DeleteEmptyContainer && newC.N() == 0 {
+					err = tx.RemoveContainer(name, itrKey)
+					if err != nil {
+						return
+					}
+					continue
+				}
+				err = tx.PutContainer(name, itrKey, newC)
+				if err != nil {
+					return
+				}
+				continue
+			}
+		} else {
+			// setting bits
+
+			existN := oldC.N()
+			if existN == roaring.MaxContainerVal+1 {
+				// completely full container already, set will do nothing. so changed of 0 default is perfect.
+				continue
+			}
+			if existN == 0 {
+				// can nsynth be zero? No, because of the continue/invariant above where nsynth > 0
+				changed += nsynth
+				rowSet[currRow] += nsynth
+				err = tx.PutContainer(name, itrKey, synthC)
+				if err != nil {
+					return
+				}
+				continue
+			}
+
+			newC := oldC.UnionInPlace(synthC)
+
+			if roaring.ContainerType(newC) == containerBitmap {
+				newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it.
+			}
+			if newC.N() != existN {
+				changes := int(newC.N() - existN)
+				changed += changes
+				rowSet[currRow] += changes
+
+				err = tx.PutContainer(name, itrKey, newC)
+				if err != nil {
+					panicOn(err)
+					return
+				}
+				continue
+			}
+		}
+	}
+	return
+}
diff --git a/tx.go b/tx.go
index ff544aca7..cc78b501f 100644
--- a/tx.go
+++ b/tx.go
@@ -993,8 +993,7 @@ func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, st
 func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {}
 
 func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
-	// TODO: Implement RBFTX.ImportRoaringBits"
-	return 0, make(map[uint64]int), nil
+	return tx.tx.ImportRoaringBits(rbfName(field, view, shard), rit, clear, log, rowSize, data)
 }
 
 func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
diff --git a/txfactory.go b/txfactory.go
index 6120efe9f..de2e90c25 100644
--- a/txfactory.go
+++ b/txfactory.go
@@ -141,7 +141,7 @@ func MustTxsrcToTxtype(txsrc string) txtype {
 
 // always store files in a subdir of dir. If we are having one
 // database or many can depend on name.
-func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
+func NewTxFactory(txsrc string, dir, name string, openExisting bool) (f *TxFactory, err error) {
 
 	ty := MustTxsrcToTxtype(txsrc)
 	if ty < 1 || ty > 9 {
@@ -162,14 +162,16 @@ func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
 		// enables cross-index Tx, which are important and are tested for.
 		path := dir + sep + "honeyBadger"
 
-		f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
-		// TODO(jea): figure out what the appropriate error path is here.
-		//fmt.Printf("warning: could not open badgerdb on path '%v': '%v'. For safety, we are opening a new '%v-fallback' instead\n", path, err, path+"-fallback")
-		if err != nil {
+		if openExisting {
+			f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
+			if err != nil {
+				return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
+			}
+		} else {
 			f.badgerDB, err = globalBadgerReg.newBadgerDBWrapper(path)
-		}
-		if err != nil {
-			return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
+			if err != nil {
+				return nil, errors.Wrap(err, fmt.Sprintf("cannot create new badger db. path='%v'", path))
+			}
 		}
 		// electric-fence like finding of access to mmapped data beyond
 		// transaction end time.
@@ -178,6 +180,7 @@ func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
 
 	switch ty {
 	case rbfTxn, blueGreenRBFRoaring, blueGreenRoaringRBF, blueGreenBadgerRBF, blueGreenRBFBadger:
+
 		f.rbfDB = rbf.NewDB(filepath.Join(dir, "db.rbf"))
 		if err := f.rbfDB.Open(); err != nil {
 			return nil, errors.Wrap(err, "cannot open rbf db")
@@ -279,6 +282,18 @@ func (f *TxFactory) CloseIndex(idx *Index) error {
 		return nil
 	case blueGreenRoaringBadger:
 		return nil
+
+	case blueGreenRBFRoaring:
+		_ = f.rbfDB.Close()
+		return nil
+	case blueGreenRoaringRBF:
+		return f.rbfDB.Close()
+	case blueGreenBadgerRBF:
+		return f.rbfDB.Close()
+	case blueGreenRBFBadger:
+		_ = f.rbfDB.Close()
+		return nil
+
 	}
 	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
 }
@@ -300,7 +315,7 @@ func (f *TxFactory) NewTx(o Txo) Tx {
 		if err != nil {
 			panic(err) // TODO: Add error return on NewTx()
 		}
-		return &RBFTx{tx: tx}
+		return &RBFTx{tx: tx, index: indexName}
 	case blueGreenBadgerRoaring:
 		btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
 		rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
@@ -310,37 +325,35 @@ func (f *TxFactory) NewTx(o Txo) Tx {
 		rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
 		return newBlueGreenTx(rtx, btx, f.idx)
 
-		/*
-			case blueGreenBadgerRBF:
-				btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
-				rbftx, err := f.rbfDB.Begin(o.Write)
-				if err != nil {
-					errors.Wrap(err, "rbfDB.Begin transaction errored")
-				}
-				return newBlueGreenTx(btx, rbftx, f.idx)
-			case blueGreenRBFBadger:
-				btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
-				rbftx, err := f.rbfDB.Begin(o.Write)
-				if err != nil {
-					errors.Wrap(err, "rbfDB.Begin transaction errored")
-				}
-				return newBlueGreenTx(rbftx, btx, f.idx)
+	case blueGreenBadgerRBF:
+		btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
+		rbftx, err := f.rbfDB.Begin(o.Write)
+		if err != nil {
+			panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
+		}
+		return newBlueGreenTx(btx, &RBFTx{tx: rbftx, index: indexName}, f.idx)
+	case blueGreenRBFBadger:
+		btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
+		rbftx, err := f.rbfDB.Begin(o.Write)
+		if err != nil {
+			panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
+		}
+		return newBlueGreenTx(&RBFTx{tx: rbftx, index: indexName}, btx, f.idx)
 
-			case blueGreenRBFRoaring:
-				rbftx, err := f.rbfDB.Begin(o.Write)
-				if err != nil {
-					errors.Wrap(err, "rbfDB.Begin transaction errored")
-				}
-				rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
-				return newBlueGreenTx(rbftx, rtx, f.idx)
-			case blueGreenRoaringRBF:
-				rbftx, err := f.rbfDB.Begin(o.Write)
-				if err != nil {
-					errors.Wrap(err, "rbfDB.Begin transaction errored")
-				}
-				rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
-				return newBlueGreenTx(rtx, rbftx, f.idx)
-		*/
+	case blueGreenRBFRoaring:
+		rbftx, err := f.rbfDB.Begin(o.Write)
+		if err != nil {
+			panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
+		}
+		rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
+		return newBlueGreenTx(&RBFTx{tx: rbftx, index: indexName}, rtx, f.idx)
+	case blueGreenRoaringRBF:
+		rbftx, err := f.rbfDB.Begin(o.Write)
+		if err != nil {
+			panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
+		}
+		rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
+		return newBlueGreenTx(rtx, &RBFTx{tx: rbftx, index: indexName}, f.idx)
 	}
 	panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
 }

From 394b8522d185e1082b1cfc30e6d095794421af4d Mon Sep 17 00:00:00 2001
From: Jason Aten <jason.aten@molecula.com>
Date: Fri, 31 Jul 2020 11:47:02 -0400
Subject: [PATCH 14/14] Follow roaring.Union() with optimize() to avoid overly
 large containers.

The cmd/loader is a preliminary sketch of the load testing tool.
---
 badger.go            |  26 ++++++-
 cmd/loader/loader.go | 150 ++++++++++++++++++++++++++++++++++++
 cmd/loader/vprint.go | 177 +++++++++++++++++++++++++++++++++++++++++++
 license.exceptions   |   1 +
 roaring/roaring.go   |   9 ++-
 5 files changed, 360 insertions(+), 3 deletions(-)
 create mode 100644 cmd/loader/loader.go
 create mode 100644 cmd/loader/vprint.go

diff --git a/badger.go b/badger.go
index ce28de54e..bcd95f8da 100644
--- a/badger.go
+++ b/badger.go
@@ -296,6 +296,7 @@ func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, e
 
 	opt.Compression = badgeroptions.None // turn off compression.
 	opt.ZSTDCompressionLevel = 0         // really, just in case.
+	opt.SyncWrites = true                // default is true, safe.
 
 	// MaxCacheSize docs:
 	//
@@ -1535,7 +1536,7 @@ func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, i
 				continue
 			}
 
-			newC := oldC.UnionInPlace(synthC)
+			newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers.
 
 			if roaring.ContainerType(newC) == containerBitmap {
 				newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it.
@@ -1634,6 +1635,9 @@ func fromArray16(a []uint16) []byte {
 	if len(a) == 0 {
 		return []byte{}
 	}
+	if len(a) > 4096 {
+		panic(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a)))
+	}
 	return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2]
 }
 
@@ -1650,6 +1654,9 @@ func fromInterval16(a []roaring.Interval16) []byte {
 	if len(a) == 0 {
 		return []byte{}
 	}
+	if len(a) > 2048 {
+		panic(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a)))
+	}
 	return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4]
 }
 
@@ -1769,6 +1776,23 @@ func asInts(a []uint64) (r []int) {
 	return
 }
 
+var _ = zeroKeyContainerAsString // happy linter
+
+// for debugging
+func zeroKeyContainerAsString(ct *roaring.Container) (r string) {
+	cts := roaring.NewSliceContainers()
+	cts.Put(0, ct)
+	rbm := &roaring.Bitmap{Containers: cts}
+	r = fmt.Sprintf("[%v]:", containerTypeNames[roaring.ContainerType(ct)]) + bitmapAsString(rbm)
+	return
+}
+
+var containerTypeNames = map[byte]string{
+	containerArray:  "array",
+	containerBitmap: "bitmap",
+	containerRun:    "run",
+}
+
 func bitmapAsString(rbm *roaring.Bitmap) (r string) {
 	r = "c("
 	slc := rbm.Slice()
diff --git a/cmd/loader/loader.go b/cmd/loader/loader.go
new file mode 100644
index 000000000..2a6a00ff2
--- /dev/null
+++ b/cmd/loader/loader.go
@@ -0,0 +1,150 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"time"
+	//"fmt"
+	"fmt"
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/http"
+	"io"
+	"io/ioutil"
+	gohttp "net/http"
+	//"log"
+	"os"
+	//"path/filepath"
+	//"sort"
+	"strconv"
+	"strings"
+)
+
+func UploadTar(srcFile string, client *http.InternalClient) error {
+	t0 := time.Now()
+
+	f, err := os.Open(srcFile)
+	if err != nil {
+		return (err)
+	}
+	defer f.Close()
+	var tarReader *tar.Reader
+	if strings.HasSuffix(srcFile, "gz") {
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		tarReader = tar.NewReader(gzf)
+	} else {
+		tarReader = tar.NewReader(f)
+	}
+	viewData := make(map[string][]byte)
+	//given ordered by index/field/view
+	//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
+	lastIndex := ""
+	lastField := ""
+	lastShard := uint64(0)
+	//vv("top of tar loop")
+	n := 0
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			if header != nil {
+				panic("header should not be nil on err io.EOF")
+			}
+			//submit any stuff we have left
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//	Submit(lastIndex, lastField, lastShard, request)
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
+				panicOn(err)
+				//vv("done with submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+			}
+			return nil
+		}
+		//vv("got header '%v'", header.Name)
+		n++
+		if n%500 == 0 {
+			vv("n = %v, progress, elapsed '%v'", n, time.Since(t0))
+		}
+		parts := strings.Split(header.Name, "/")
+		index := parts[0]
+		field := parts[1]
+		view := parts[3]
+		shard, err := strconv.ParseUint(parts[5], 10, 64)
+		if err != nil {
+			return err
+		}
+		// TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro.
+		if index != lastIndex || field != lastField || shard != lastShard {
+			if len(viewData) > 0 {
+				request := &pilosa.ImportRoaringRequest{
+					Views: viewData,
+				}
+				//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
+				uri := GetImportRoaringURI(lastIndex, lastShard)
+				panicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
+				viewData = make(map[string][]byte)
+				//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
+
+			}
+		}
+		roaringData, err := ioutil.ReadAll(tarReader)
+		if err != nil {
+			return err
+		}
+		if _, already := viewData[view]; already {
+			panic(fmt.Sprintf("view '%v' already present!", view))
+		}
+		viewData[view] = roaringData
+		lastIndex = index
+		lastField = field
+
+		//lastShard = shard
+		//vv("bottom of loop")
+	}
+}
+
+func main() {
+
+	host := "127.0.0.1:10101"
+	h := &gohttp.Client{}
+	c, err := http.NewInternalClient(host, h)
+	panicOn(err)
+
+	tarSrcPath := "q2.tar.gz"
+	t0 := time.Now()
+	panicOn(UploadTar(tarSrcPath, c))
+	vv("total elapsed '%v'", time.Since(t0))
+}
+
+var globURI *pilosa.URI
+
+func init() {
+	var err error
+	globURI, err = pilosa.NewURIFromHostPort("127.0.0.1", 10101)
+	panicOn(err)
+}
+
+// get correct node to go to.
+func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
+	return globURI
+}
diff --git a/cmd/loader/vprint.go b/cmd/loader/vprint.go
new file mode 100644
index 000000000..e6bea8d55
--- /dev/null
+++ b/cmd/loader/vprint.go
@@ -0,0 +1,177 @@
+// home: https://github.com/glyerine/vprint
+// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
+// License: MIT
+//
+// MIT License
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"runtime"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
+const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
+
+// for tons of debug output
+var VerboseVerbose bool = false
+
+// convience functions for . import
+var pp = PP
+var vv = VV
+
+var panicOn = PanicOn
+
+func init() {
+	// keeper linter happy
+	_ = pp
+	_ = vv
+}
+
+func PanicOn(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func PP(format string, a ...interface{}) {
+	if VerboseVerbose {
+		TSPrintf(format, a...)
+	}
+}
+
+func VV(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+func AlwaysPrintf(format string, a ...interface{}) {
+	TSPrintf(format, a...)
+}
+
+var tsPrintfMut sync.Mutex
+
+// time-stamped printf
+func TSPrintf(format string, a ...interface{}) {
+	tsPrintfMut.Lock()
+	Printf("\n%s %s ", FileLine(3), ts())
+	Printf(format+"\n", a...)
+	tsPrintfMut.Unlock()
+}
+
+// get timestamp for logging purposes
+func ts() string {
+	return time.Now().Format(RFC3339UsecTz0)
+}
+
+// so we can multi write easily, use our own printf
+var OurStdout io.Writer = os.Stdout
+
+// Printf formats according to a format specifier and writes to standard output.
+// It returns the number of bytes written and any write error encountered.
+func Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(OurStdout, format, a...)
+}
+
+func FileLine(depth int) string {
+	_, fileName, fileLine, ok := runtime.Caller(depth)
+	var s string
+	if ok {
+		s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
+	} else {
+		s = ""
+	}
+	return s
+}
+
+func stack() string {
+	return string(debug.Stack())
+}
+
+func FileExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return false
+	}
+	return true
+}
+
+func DirExists(name string) bool {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return false
+	}
+	if fi.IsDir() {
+		return true
+	}
+	return false
+}
+
+func FileSize(name string) (int64, error) {
+	fi, err := os.Stat(name)
+	if err != nil {
+		return -1, err
+	}
+	return fi.Size(), nil
+}
+
+// Caller returns the name of the calling function.
+func Caller(upStack int) string {
+	// elide ourself and runtime.Callers
+	target := upStack + 2
+
+	pc := make([]uintptr, target+2)
+	n := runtime.Callers(0, pc)
+
+	f := runtime.Frame{Function: "unknown"}
+	if n > 0 {
+		frames := runtime.CallersFrames(pc[:n])
+		for i := 0; i <= target; i++ {
+			contender, more := frames.Next()
+			if i == target {
+				f = contender
+			}
+			if !more {
+				break
+			}
+		}
+	}
+	return f.Function
+}
+
+// happy linter:
+var _ = DirExists
+var _ = FileExists
+var _ = Caller
+var _ = stack
+var _ = RFC3339MsecTz0
+var _ = RFC3339UsecTz0
+var _ = AlwaysPrintf
+var _ = FileSize
diff --git a/license.exceptions b/license.exceptions
index 67767e1e2..ace441ae6 100644
--- a/license.exceptions
+++ b/license.exceptions
@@ -11,3 +11,4 @@
 ./logger/filewriter_test.go
 ./vprint.go
 ./rbf/vprint.go
+./cmd/loader/vprint.go
diff --git a/roaring/roaring.go b/roaring/roaring.go
index 41ff21a17..9219b01ef 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -4368,6 +4368,7 @@ func unionArrayArray(a, b *Container) *Container {
 			break
 		}
 	}
+	// note: len(output) CAN be > 4096
 	return NewContainerArray(output)
 }
 
@@ -6865,8 +6866,12 @@ func ConvertRunToBitmap(c *Container) {
 func Optimize(c *Container) {
 	c.optimize()
 }
-func Union(a, b *Container) *Container {
-	return union(a, b)
+func Union(a, b *Container) (c *Container) {
+	c = union(a, b)
+	// c can be have arrays that are too big, and need
+	// to be optimized into raw bitmaps.
+	c.optimize()
+	return c
 }
 
 func Difference(a, b *Container) *Container {