Merge branch 'master' into fix-rbf-race

.circleci/config.yml

@@ -186,6 +186,7 @@ workflows:
           matrix:
             parameters:
               golang_version: ["1.14", "1.13"]
+          resource_class: large
           requires:
             - setup
           filters:

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
+.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@@ -43,6 +43,33 @@ test:
 test-race:
 	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v
 
+testv: topt testvsub
+
+testv-race: topt-race testvsub-race
+
+# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
+#            avoiding go -test ./... "package list mode" which doesn't give output
+#            until the test run finishes. Package list mode makes it hard to
+#            find which test is hung/deadlocked.
+#
+testvsub:
+	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i"; \
+           cd $$i; pwd; \
+           go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v || break; \
+           echo; echo "999 done testing subpkg $$i"; \
+           cd ..; \
+        done
+
+testvsub-race:
+	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i -race"; \
+           cd $$i; pwd; \
+           go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race || break; \
+           echo; echo "999 done testing subpkg $$i -race"; \
+           cd ..; \
+        done
+
 bench:
 	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
 

api.go

@@ -1114,9 +1114,6 @@ func (api *API) ImportAtomicRecord(ctx context.Context, req *AtomicRecord, opts
 	return tx.Commit()
 }
 
-// This is a hide your face ugly hack, forced upon
-// us by the horrible invention of function based options
-// by the usually brilliant Rob Pike. - JEA
 func addClearToImportOptions(opts []ImportOption) []ImportOption {
 	var opt ImportOptions
 	for _, o := range opts {

api_test.go

@@ -59,8 +59,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 	t.Run("ImportColumnAttrs", func(t *testing.T) {
 		ctx := context.Background()
 		indexName := "i"
@@ -184,8 +184,8 @@ func TestAPI_Import(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 
 	t.Run("RowIDColumnKey", func(t *testing.T) {
 		ctx := context.Background()
@@ -293,8 +293,8 @@ func TestAPI_ImportValue(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 
 	t.Run("ValColumnKey", func(t *testing.T) {
 		ctx := context.Background()
@@ -492,7 +492,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
 	//  3. verifiy the clear is done.
 	//  repeat for ImportValueRequest and ImportValues()
 
-	m0 := c[0]
+	m0 := c.GetNode(0)
 	m0api := m0.API
 
 	ctx := context.Background()

audit.go

@@ -0,0 +1,25 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+var NewAuditor func() testhook.Auditor = NewNopAuditor
+
+func NewNopAuditor() testhook.Auditor {
+	return testhook.NewNopAuditor()
+}

audit_internal_test.go

@@ -0,0 +1,52 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorViewHooks struct{}
+type auditorFragmentHooks struct{}
+
+// static type checks
+var _ testhook.RegistryHookLive = &auditorViewHooks{}
+var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
+
+func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("view %s still open", o.(*view).name)
+	}
+	return nil
+}
+
+func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("fragment %s still open", o.(*fragment).path)
+	}
+	return nil
+}
+
+func GetInternalTestHooks() testhook.RegistryHooks {
+	return map[reflect.Type]testhook.RegistryHook{
+		reflect.TypeOf((*view)(nil)):     &auditorViewHooks{},
+		reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
+	}
+}

audit_test.go

@@ -0,0 +1,107 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// AuditLeaksOn is a global switch to turn on resource
+// leak checking at the end of a test run.
+var AuditLeaksOn = true
+
+// for tests, we use a single shared auditor used by all of the holders.
+var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorIndexHooks struct{}
+type auditorFieldHooks struct{}
+type auditorHolderHooks struct{}
+
+// static type checking
+var _ testhook.RegistryHookLive = &auditorIndexHooks{}
+var _ testhook.RegistryHookLive = &auditorFieldHooks{}
+var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
+var _ testhook.RegistryHookLive = &auditorHolderHooks{}
+
+var testHooks = map[reflect.Type]testhook.RegistryHook{
+	reflect.TypeOf((*pilosa.Index)(nil)):  &auditorIndexHooks{},
+	reflect.TypeOf((*pilosa.Field)(nil)):  &auditorFieldHooks{},
+	reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
+}
+
+func init() {
+	if !AuditLeaksOn {
+		return
+	}
+	for k, v := range pilosa.GetInternalTestHooks() {
+		testHooks[k] = v
+	}
+	testhook.RegisterPreTestHook(func() error {
+		pilosa.NewAuditor = NewTestAuditor
+		return nil
+	})
+	testhook.RegisterPostTestHook(func() error {
+		err, errs := globalTestAuditor.FinalCheck()
+		if err != nil {
+			for i, e := range errs {
+				fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
+			}
+		}
+		return err
+	})
+}
+
+func NewTestAuditor() testhook.Auditor {
+	return globalTestAuditor
+}
+
+func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
+	}
+	return nil
+}
+
+func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
+	}
+	return nil
+}
+
+func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
+	path := o.(*pilosa.Holder).Path
+	if path == "" {
+		fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
+			ent.Stack)
+	} else {
+		os.RemoveAll(o.(*pilosa.Holder).Path)
+	}
+	return err
+}
+
+func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path)
+	}
+	return nil
+}

badger.go

@@ -32,6 +32,7 @@ import (
 	badger "github.com/dgraph-io/badger/v2"
 	badgeroptions "github.com/dgraph-io/badger/v2/options"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/txkey"
 	"github.com/pkg/errors"
 )
@@ -329,6 +330,7 @@ func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, e
 		halt:   halt,
 		hasher: NewBlake3Hasher(),
 	}
+	_ = testhook.Opened(NewAuditor(), w, nil)
 	r.unprotectedRegister(w)
 
 	w.startStack = stack()
@@ -504,6 +506,7 @@ func (w *BadgerDBWrapper) Close() (err error) {
 		close(w.halt)
 		w.closed = true
 	}
+	_ = testhook.Closed(NewAuditor(), w, nil)
 	return w.db.Close()
 }
 

badger_test.go

@@ -18,12 +18,8 @@
 // See https://github.com/dgraph-io/badger/issues/1384 for any progress.
 // What we see is that the value-log allocations immediately run out of
 // memory. So we turn off 386 with a build tag to keep the .circleci happy.
-//
-// gendebug_test will have a TestMain if build tag generationdebug is on,
-// so we avoid conflicting with that debug scenario.
 
 // +build !386
-// +build !generationdebug
 
 package pilosa
 
@@ -36,10 +32,16 @@ import (
 
 	"github.com/dgraph-io/badger/v2"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
+	"github.com/pkg/errors"
 )
 
 var _ = &roaring.Bitmap{}
 
+func init() {
+	testhook.RegisterPostTestHook(reportTestBadgersNeedingClose)
+}
+
 // helpers, each runs their own new txn, and commits if a change/delete
 // was made. The txn is rolled back if it is just viewing the data.
 
@@ -1639,26 +1641,20 @@ func BenchmarkBadger_Write(b *testing.B) {
 	*/
 }
 
-func reportTestBadgersNeedingClose() {
+func reportTestBadgersNeedingClose() error {
 	globalBadgerReg.mu.Lock()
 	defer globalBadgerReg.mu.Unlock()
 	n := len(globalBadgerReg.mp)
-	if n > 0 {
-		AlwaysPrintf("*** these badgers are still open (n=%v):", n)
-		i := 0
-		for w := range globalBadgerReg.mp {
-			AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack)
-			i++
-		}
+	if n == 0 {
+		return nil
 	}
-}
-
-var _ = reportTestBadgersNeedingClose // happy linter
-
-func TestMain(m *testing.M) {
-	ret := m.Run()
-	//reportTestBadgersNeedingClose()
-	os.Exit(ret)
+	AlwaysPrintf("*** these badgers are still open (n=%v):", n)
+	i := 0
+	for w := range globalBadgerReg.mp {
+		AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack)
+		i++
+	}
+	return errors.New("unclosed badgers, contact Animal Control")
 }
 
 /*

blake3_test.go

@@ -21,6 +21,8 @@ import (
 	"testing"
 
 	"encoding/hex"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestBlake3Hasher(t *testing.T) {
@@ -49,7 +51,7 @@ func TestCryptoRandInt64(t *testing.T) {
 }
 
 func TestHashOfDir(t *testing.T) {
-	dir, err := ioutil.TempDir(".", "TestHashOfDir-dir")
+	dir, err := testhook.TempDir(t, "TestHashOfDir-dir")
 	panicOn(err)
 	b := dir + sep + "A" + sep + "B"
 	c := dir + sep + "A" + sep + "C"
@@ -59,7 +61,6 @@ func TestHashOfDir(t *testing.T) {
 	panicOn(ioutil.WriteFile(b+sep+"b_content", bmessage, 0644))
 	cmessage := []byte("hello C\n")
 	panicOn(ioutil.WriteFile(c+sep+"c_content", cmessage, 0644))
-	defer os.RemoveAll(dir)
 	hsh := HashOfDir(dir)
 
 	c2message := []byte("hello C2\n")

cluster_internal_test.go

@@ -17,7 +17,6 @@ package pilosa
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -35,6 +34,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pilosa/pilosa/v2/logger"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -91,14 +91,17 @@ func TestFragCombos(t *testing.T) {
 }
 
 // newIndexWithTempPath returns a new instance of Index.
-func newIndexWithTempPath(name string) *Index {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
+func newIndexWithTempPath(tb testing.TB, name string) *Index {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}
 	h := NewHolder(DefaultPartitionN)
 	h.Path = path
 	index, err := h.CreateIndex(name, IndexOptions{})
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	if err != nil {
 		panic(err)
 	}
@@ -158,7 +161,7 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node2)
 	c5.addNodeBasicSorted(node3)
 
-	idx := newIndexWithTempPath("i")
+	idx := newIndexWithTempPath(t, "i")
 	defer idx.Close()
 
 	// Obtain transaction.
@@ -398,7 +401,7 @@ func TestHasher(t *testing.T) {
 
 // Ensure ContainsShards can find the actual shard list for node and index.
 func TestCluster_ContainsShards(t *testing.T) {
-	c := NewTestCluster(5)
+	c := NewTestCluster(t, 5)
 	c.ReplicaN = 3
 	shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2])
 
@@ -544,7 +547,7 @@ func TestCluster_Coordinator(t *testing.T) {
 }
 
 func TestCluster_Topology(t *testing.T) {
-	c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
+	c1 := NewTestCluster(t, 1) // automatically creates Node{ID: "node0"}
 
 	uri0 := NewTestURIFromHostPort("host0", 0)
 	uri1 := NewTestURIFromHostPort("host1", 0)
@@ -592,7 +595,7 @@ func TestCluster_Topology(t *testing.T) {
 func TestCluster_ResizeStates(t *testing.T) {
 
 	t.Run("Single node, no data", func(t *testing.T) {
-		tc := NewClusterCluster(1)
+		tc := NewClusterCluster(t, 1)
 
 		// Open TestCluster.
 		if err := tc.Open(); err != nil {
@@ -622,7 +625,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Single node, in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -654,7 +657,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Single node, not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -683,7 +686,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, no data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -725,7 +728,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -774,7 +777,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, with data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -927,7 +930,7 @@ func TestAE(t *testing.T) {
 // Ensures that coordinator can be changed.
 func TestCluster_UpdateCoordinator(t *testing.T) {
 	t.Run("UpdateCoordinator", func(t *testing.T) {
-		c := NewTestCluster(2)
+		c := NewTestCluster(t, 2)
 
 		oldNode := c.nodes[0]
 		newNode := c.nodes[1]

ctl/export_test.go

@@ -46,7 +46,7 @@ func TestExportCommand_Validation(t *testing.T) {
 func TestExportCommand_Run(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)

ctl/import_test.go

@@ -73,7 +73,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -102,7 +102,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -135,7 +135,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -177,7 +177,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -219,7 +219,7 @@ func TestImportCommand_RunKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -271,8 +271,8 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	c := test.MustRunCluster(t, 2)
 	defer c.Close()
-	cmd0 := c[0]
-	cmd1 := c[1]
+	cmd0 := c.GetNode(0)
+	cmd1 := c.GetNode(1)
 
 	host0 := cmd0.API.Node().URI.HostPort()
 	host1 := cmd1.API.Node().URI.HostPort()
@@ -339,7 +339,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -365,7 +365,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 func TestImportCommand_InvalidFile(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -453,7 +453,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
 func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -529,7 +529,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))

diagnostics_internal_test.go

@@ -29,6 +29,7 @@ import (
 func TestDiagnosticsClient(t *testing.T) {
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)
@@ -121,6 +122,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
 			t.Fatalf("couldn't encode version response: %v", err)
 		}
 	}))
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector("localhost:10101")
@@ -158,6 +160,7 @@ func compareJSON(a, b []byte) (bool, error) {
 func BenchmarkDiagnostics(b *testing.B) {
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)

executor.go

@@ -29,6 +29,7 @@ import (
 	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/shardwidth"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 )
@@ -100,6 +101,7 @@ func newExecutor(opts ...executorOption) *executor {
 	// the few tests we've done at scale with concurrent query
 	// workloads. Possible that it could be smaller.
 	e.work = make(chan job, e.workerPoolSize)
+	_ = testhook.Opened(NewAuditor(), e, nil)
 	for i := 0; i < e.workerPoolSize; i++ {
 		e.workersWG.Add(1)
 		go func() {
@@ -114,6 +116,7 @@ func (e *executor) Close() error {
 	e.workMu.Lock()
 	defer e.workMu.Unlock()
 	e.shutdown = true
+	_ = testhook.Closed(NewAuditor(), e, nil)
 	close(e.work)
 	e.workersWG.Wait()
 	return nil

executor_internal_test.go

@@ -18,24 +18,25 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"strconv"
 	"strings"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestExecutor_TranslateGroupByCall(t *testing.T) {
 	holder := NewHolder(DefaultPartitionN)
+	defer holder.Close()
 
-	cluster := NewTestCluster(1)
+	cluster := NewTestCluster(t, 1)
 
 	e := &executor{
 		Holder:  holder,
 		Cluster: cluster,
 	}
-	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
+	e.Holder.Path, _ = testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
 	err := e.Holder.Open()
 	if err != nil {
 		t.Fatalf("opening holder: %v", err)
@@ -139,9 +140,9 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) {
 
 	e := &executor{
 		Holder:  holder,
-		Cluster: NewTestCluster(1),
+		Cluster: NewTestCluster(t, 1),
 	}
-	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
+	e.Holder.Path, _ = testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
 	if err := e.Holder.Open(); err != nil {
 		t.Fatalf("opening holder: %v", err)
 	}

field.go

@@ -35,6 +35,7 @@ import (
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -607,9 +608,11 @@ func (f *Field) Open() error {
 		return err
 	}
 
+	_ = testhook.Opened(f.holder.Auditor, f, nil)
 	f.holder.Logger.Debugf("successfully opened field index/field: %s/%s", f.index, f.name)
 	return nil
 }
+
 func blockingWriteAvailableShards(fieldPath string, availableShardBytes []byte) {
 	path := filepath.Join(fieldPath, ".available.shards")
 	// Create a temporary file to save to.
@@ -972,12 +975,14 @@ func (f *Field) applyOptions(opt FieldOptions) error {
 func (f *Field) Close() error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	defer func() {
+		_ = testhook.Closed(f.holder.Auditor, f, nil)
+	}()
 	// Shutdown the available shards writer
 	if f.doneChan != nil {
-		f.doneChan <- struct{}{}
+		close(f.doneChan)
 		f.wg.Wait()
 		close(f.availableShardChan)
-		close(f.doneChan)
 		f.availableShardChan = nil
 		f.doneChan = nil
 	}

field_internal_test.go

@@ -16,7 +16,6 @@ package pilosa
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math"
 	"os"
 	"path/filepath"
@@ -28,6 +27,7 @@ import (
 
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure a bsiGroup can adjust to its baseValue.
@@ -197,11 +197,13 @@ func TestField_DeleteView(t *testing.T) {
 // TestField represents a test wrapper for Field.
 type TestField struct {
 	*Field
+	parent *Index
+	tb     testing.TB
 }
 
 // NewTestField returns a new instance of TestField d/0.
 func NewTestField(t *testing.T, opts FieldOption) *TestField {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-field-")
+	path, err := testhook.TempDirInDir(t, *TempDir, "pilosa-field-")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -211,20 +213,20 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
 	if err != nil {
 		panic(err)
 	}
-	field, err := NewField(h, path, "i", "f", opts)
+	field, err := idx.CreateField("f", opts)
 	if err != nil {
 		t.Fatal(err)
 	}
-	field.idx = idx
-	return &TestField{Field: field}
+	tf := &TestField{Field: field, parent: idx, tb: t}
+	testhook.Cleanup(t, func() {
+		h.Close()
+	})
+	return tf
 }
 
 // OpenField returns a new, opened field at a temporary path.
 func OpenField(t *testing.T, opts FieldOption) *TestField {
 	f := NewTestField(t, opts)
-	if err := f.Open(); err != nil {
-		t.Fatal(err)
-	}
 	return f
 }
 
@@ -239,27 +241,16 @@ func (f *TestField) Close() error {
 
 // Reopen closes the index and reopens it.
 func (f *TestField) Reopen() error {
-	var err error
-	if err := f.Field.Close(); err != nil {
+	name := f.Field.Name()
+	if err := f.parent.Close(); err != nil {
+		f.parent = nil
 		return err
 	}
-
-	path, index, name := f.Path(), f.Index(), f.Name()
-	h := NewHolder(DefaultPartitionN)
-	h.Path = path
-	idx, err := h.CreateIndex(index, IndexOptions{})
-	if err != nil {
-		return err
-	}
-	f.Field, err = NewField(h, path, index, name, OptFieldTypeDefault())
-	if err != nil {
-		return err
-	}
-	f.Field.idx = idx
-
-	if err := f.Open(); err != nil {
+	if err := f.parent.Open(false); err != nil {
+		f.parent = nil
 		return err
 	}
+	f.Field = f.parent.Field(name)
 	return nil
 }
 

field_test.go

@@ -15,7 +15,6 @@
 package pilosa_test
 
 import (
-	"io/ioutil"
 	"math"
 	"testing"
 
@@ -23,6 +22,7 @@ import (
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 var panicOn = pilosa.PanicOn
@@ -30,7 +30,7 @@ var panicOn = pilosa.PanicOn
 // Ensure a field can set & read a bsiGroup value.
 func TestField_SetValue(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
@@ -67,7 +67,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("Overwrite", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
@@ -103,7 +103,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("ErrBSIGroupNotFound", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
@@ -121,7 +121,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
@@ -139,7 +139,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
@@ -158,7 +158,7 @@ func TestField_SetValue(t *testing.T) {
 }
 
 func TestField_NameRestriction(t *testing.T) {
-	path, err := ioutil.TempDir("", "pilosa-field-")
+	path, err := testhook.TempDir(t, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
@@ -190,7 +190,7 @@ func TestField_NameValidation(t *testing.T) {
 		"charact23112345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901",
 	}
 
-	path, err := ioutil.TempDir("", "pilosa-field-")
+	path, err := testhook.TempDir(t, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
@@ -210,7 +210,7 @@ func TestField_NameValidation(t *testing.T) {
 
 // Ensure can update and delete available shards.
 func TestField_AvailableShards(t *testing.T) {
-	idx := test.MustOpenIndex()
+	idx := test.MustOpenIndex(t)
 	defer idx.Close()
 
 	f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
@@ -253,7 +253,7 @@ func TestField_AvailableShards(t *testing.T) {
 
 func TestField_ClearValue(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))

fragment.go

@@ -45,6 +45,7 @@ import (
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/shardwidth"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 )
@@ -170,15 +171,6 @@ type fragment struct {
 	stats stats.StatsClient
 
 	bitmapInfo *roaring.BitmapInfo
-
-	// txTestingOnly: this looks gross.
-	// Nonetheless, it allowed us to
-	// integrate Tx into the
-	// fragment_internal_test.go suite
-	// and not break the world all at once.
-	//
-	// Only for testing, obviously.
-	txTestingOnly Tx
 }
 
 // newFragment returns a new instance of Fragment.
@@ -267,6 +259,7 @@ func (f *fragment) Open() error {
 	}
 	f.open = true
 
+	_ = testhook.Opened(f.holder.Auditor, f, nil)
 	f.holder.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
 	return nil
 }
@@ -513,6 +506,9 @@ func (f *fragment) openCache() error {
 func (f *fragment) Close() error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	defer func() {
+		_ = testhook.Closed(f.holder.Auditor, f, nil)
+	}()
 	for f.snapshotPending {
 		f.snapshotCond.Wait()
 	}

gendebug_test.go

@@ -17,23 +17,31 @@
 package pilosa
 
 import (
+	"errors"
 	"fmt"
-	"os"
-	"testing"
+	"runtime"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
-func examineResults() {
-	results := reportGenerations()
+func examineResults() error {
+	runtime.GC()
+	stats, results := reportGenerations()
+	if len(stats) > 0 {
+		fmt.Printf("generation stats: %s\n", stats)
+	}
+	if len(results) == 0 {
+		return nil
+	}
 	if len(results) > 0 {
 		fmt.Printf("generations:\n")
 		for _, res := range results {
 			fmt.Printf("  %s\n", res)
 		}
 	}
+	return errors.New("outstanding generations detected")
 }
 
-func TestMain(m *testing.M) {
-	ret := m.Run()
-	examineResults()
-	os.Exit(ret)
+func init() {
+	testhook.RegisterPostTestHook(examineResults)
 }

generation_debug.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"math/rand"
 	"runtime"
+	"runtime/debug"
 	"sort"
 	"sync"
 	"time"
@@ -29,6 +30,7 @@ const generationDebug = true
 
 type lifespan struct {
 	from, to, finalized time.Time
+	stack               []byte
 }
 
 var knownGenerations map[string]lifespan
@@ -36,32 +38,49 @@ var knownGenerationLock sync.Mutex
 
 var timeZero time.Time
 
+var generationDebugVerbose bool
+
+// History reports the finalized/dead/created status of a span which we think
+// is in some way in error. It's shared between a couple of places.
+func (span *lifespan) History() string {
+	dead := "not dead"
+	finalized := "not finalized"
+	if span.finalized != timeZero {
+		finalized = fmt.Sprintf("finalized at %v", span.finalized)
+	}
+	if span.to != timeZero {
+		dead = fmt.Sprintf("dead at %v", span.to)
+	}
+	return fmt.Sprintf("%s, %s, created at %v at %s", dead, finalized, span.from, span.stack)
+}
+
+func (span *lifespan) reportHistory(reason string, id string) string {
+	return fmt.Sprintf("%s %s: %s", id, reason, span.History())
+}
+
 func registerGeneration(id string) string {
 	knownGenerationLock.Lock()
 	defer knownGenerationLock.Unlock()
 	if knownGenerations == nil {
 		knownGenerations = make(map[string]lifespan)
 	}
-	newSpan := lifespan{from: time.Now()}
+	newSpan := lifespan{from: time.Now(), stack: debug.Stack()}
 	origId := id
 
 	// if you have more than 65k of the same file open, maybe you have bigger
 	// problems than this.
 	for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] {
 		suffix := fmt.Sprintf("::%04x", rand.Int63n(65536))
-		if span.finalized != timeZero {
-			fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v, finalized %v\n",
-				id, suffix, span.from, span.to, span.finalized)
-		} else {
-			if span.to != timeZero {
-				fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v\n", id, suffix, span.from, span.to)
-			} else {
-				fmt.Printf("new generation %s: adding %s, already exists, created %v", id, suffix, span.from)
-			}
+		if generationDebugVerbose {
+			history := span.History()
+			fmt.Printf("new generation: adding suffix %s, previous %s\n",
+				suffix, history)
 		}
 		id = origId + suffix
 	}
-	fmt.Printf("new generation %s\n", id)
+	if generationDebugVerbose {
+		fmt.Printf("new generation %s\n", id)
+	}
 	knownGenerations[id] = newSpan
 	return id
 }
@@ -75,8 +94,7 @@ func endGeneration(id string) {
 		panic(oops)
 	}
 	if span.finalized != timeZero || span.to != timeZero {
-		oops := fmt.Sprintf("ending generation %s: already died at %v, finalized at %v", id, span.to, span.finalized)
-		panic(oops)
+		panic(span.reportHistory("ending generation", id))
 	}
 	span.to = time.Now()
 	knownGenerations[id] = span
@@ -108,39 +126,25 @@ func finalizeGeneration(id string) {
 		panic(oops)
 	}
 	if span.finalized != timeZero {
-		var oops string
-		if span.to != timeZero {
-			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, but not dead", id, span.finalized)
-		} else {
-			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, dead at %v", id, span.finalized, span.to)
-		}
-		panic(oops)
+		panic(span.reportHistory("finalizing", id))
 	}
 	span.finalized = time.Now()
 	knownGenerations[id] = span
 }
 
-func reportGenerations() []string {
+func reportGenerations() (stats string, surviving []string) {
 	runtime.GC()
 	knownGenerationLock.Lock()
 	defer knownGenerationLock.Unlock()
-	var surviving []string
 	times := make([]int64, 0, len(knownGenerations))
 	for id, span := range knownGenerations {
-		if span.to == timeZero {
-			if span.finalized == timeZero {
-				surviving = append(surviving, fmt.Sprintf("%s: %v, not ended or finalized", id, span.from))
-			} else {
-				surviving = append(surviving, fmt.Sprintf("%s: %v, finalized %v, not ended", id, span.from, span.finalized))
-			}
+		if span.to == timeZero || span.finalized == timeZero {
+			surviving = append(surviving, span.reportHistory("surviving", id))
 		} else {
-			if span.finalized == timeZero {
-				surviving = append(surviving, fmt.Sprintf("%s: %v to %v, not finalized", id, span.from, span.to))
-			} else {
-				times = append(times, int64(span.finalized.Sub(span.to)))
-			}
+			times = append(times, int64(span.finalized.Sub(span.to)))
 		}
 	}
+	stats = "no recorded finalized spans"
 	if len(times) > 0 {
 		sort.Slice(times, func(i, j int) bool { return times[i] < times[j] })
 		var total int64
@@ -153,8 +157,8 @@ func reportGenerations() []string {
 		p90 = times[(len(times)*9)/10]
 		p99 = times[(len(times)*99)/100]
 		worst = times[len(times)-1]
-		surviving = append(surviving, fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
-			len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst)))
+		stats = fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
+			len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst))
 	}
-	return surviving
+	return stats, surviving
 }

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
-	github.com/glycerine/lmdb-go v1.9.26
+	github.com/glycerine/lmdb-go v1.9.27
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.2.0
 	github.com/golang/protobuf v1.3.3

go.sum

@@ -55,8 +55,8 @@ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy
 github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E=
 github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y=
-github.com/glycerine/lmdb-go v1.9.26 h1:4aIiCQhg5fLChuZuATDHD4Lr6y9CdEHLtvROkzCZKIg=
-github.com/glycerine/lmdb-go v1.9.26/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
+github.com/glycerine/lmdb-go v1.9.27 h1:k20zfiumwC/E1g/MYIzZ2GkhOH0hicDfEXa0KUjWLjY=
+github.com/glycerine/lmdb-go v1.9.27/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=

gossip/gossip.go

@@ -104,6 +104,7 @@ func (g *memberSet) Open() (err error) {
 // Close attempts to gracefully leave the cluster, and finally calls shutdown
 // after (at most) a timeout period.
 func (g *memberSet) Close() error {
+	g.eventReceiver.Close()
 	leaveErr := g.memberlist.Leave(5 * time.Second)
 	shutdownErr := g.memberlist.Shutdown()
 	if leaveErr != nil || shutdownErr != nil {
@@ -366,8 +367,9 @@ func (g *memberSet) MergeRemoteState(buf []byte, join bool) {
 // Care must be taken that events are processed in a timely manner from
 // the channel, since this delegate will block until an event can be sent.
 type eventReceiver struct {
-	ch   chan memberlist.NodeEvent
-	papi *pilosa.API
+	ch     chan memberlist.NodeEvent
+	closed chan struct{}
+	papi   *pilosa.API
 
 	logger logger.Logger
 }
@@ -376,6 +378,7 @@ type eventReceiver struct {
 func newEventReceiver(logger logger.Logger, papi *pilosa.API) *eventReceiver {
 	ger := &eventReceiver{
 		ch:     make(chan memberlist.NodeEvent, 1),
+		closed: make(chan struct{}),
 		logger: logger,
 		papi:   papi,
 	}
@@ -389,7 +392,10 @@ func (g *eventReceiver) NotifyJoin(n *memberlist.Node) {
 	n2.Meta = make([]byte, len(n.Meta))
 	copy(n2.Meta, n.Meta)
 
-	g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}
+	select {
+	case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}:
+	case <-g.closed:
+	}
 }
 
 func (g *eventReceiver) NotifyLeave(n *memberlist.Node) {
@@ -398,7 +404,10 @@ func (g *eventReceiver) NotifyLeave(n *memberlist.Node) {
 	n2.Meta = make([]byte, len(n.Meta))
 	copy(n2.Meta, n.Meta)
 
-	g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}
+	select {
+	case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}:
+	case <-g.closed:
+	}
 }
 
 func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) {
@@ -407,13 +416,25 @@ func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) {
 	n2.Meta = make([]byte, len(n.Meta))
 	copy(n2.Meta, n.Meta)
 
-	g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}
+	select {
+	case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}:
+	case <-g.closed:
+	}
+}
+
+func (g *eventReceiver) Close() {
+	close(g.closed)
 }
 
 func (g *eventReceiver) listen() {
 	var nodeEventType pilosa.NodeEventType
 	for {
-		e := <-g.ch
+		var e memberlist.NodeEvent
+		select {
+		case <-g.closed:
+			return
+		case e = <-g.ch:
+		}
 		switch e.Event {
 		case memberlist.NodeJoin:
 			nodeEventType = pilosa.NodeJoin

handler.go

@@ -127,7 +127,7 @@ type ImportValueRequest struct {
 	Values       []int64 // e.g. temperature, humidity, barometric pressure
 	FloatValues  []float64
 	StringValues []string
-	Clear        bool // only works for ImportAtomicRecord() at the moment.
+	Clear        bool
 }
 
 // AtomicRecord applies all its Ivr and Ivr atomically, in a Tx.
@@ -214,7 +214,7 @@ type ImportRequest struct {
 	RowKeys        []string
 	ColumnKeys     []string
 	Timestamps     []int64
-	Clear          bool // only works for ImportAtomicRecord() at the moment.
+	Clear          bool
 }
 
 // ValidateWithTimestamp ensures that the payload of the request is valid.

holder.go

@@ -32,6 +32,7 @@ import (
 	"github.com/pilosa/pilosa/v2/logger"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 	uuid "github.com/satori/go.uuid"
@@ -105,6 +106,8 @@ type Holder struct {
 	opening bool
 
 	Opts HolderOpts
+
+	Auditor testhook.Auditor
 }
 
 type HolderOpts struct {
@@ -165,7 +168,7 @@ func (lc *lockedChan) Recv() {
 
 // NewHolder returns a new instance of Holder.
 func NewHolder(partitionN int) *Holder {
-	return &Holder{
+	h := &Holder{
 		partitionN: partitionN,
 		indexes:    make(map[string]*Index),
 		closing:    make(chan struct{}),
@@ -188,7 +191,11 @@ func NewHolder(partitionN int) *Holder {
 		Logger: logger.NopLogger,
 
 		SnapshotQueue: defaultSnapshotQueue,
+
+		Auditor: NewAuditor(),
 	}
+	_ = testhook.Created(h.Auditor, h, nil)
+	return h
 }
 
 type HolderInfo struct {
@@ -535,6 +542,8 @@ func (h *Holder) Open() error {
 			err = index.Open(false)
 		}
 		if err != nil {
+			// FIXME: The holder shouldn't be responsible for closing these, probably.
+			_ = index.Txf.CloseDB()
 			if err == ErrName {
 				h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err)
 				continue
@@ -559,6 +568,8 @@ func (h *Holder) Open() error {
 
 	h.opened.Close()
 
+	_ = testhook.Opened(h.Auditor, h, nil)
+
 	return nil
 }
 
@@ -627,6 +638,12 @@ func (h *Holder) Close() error {
 	h.opened.mu.Lock()
 	h.opened.ch = make(chan struct{})
 	h.opened.mu.Unlock()
+	if h.SnapshotQueue != nil {
+		h.SnapshotQueue.Stop()
+		h.SnapshotQueue = nil
+	}
+
+	_ = testhook.Closed(h.Auditor, h, nil)
 
 	return nil
 }

holder_internal_test.go

@@ -16,9 +16,10 @@ package pilosa
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 type testHolderOperator struct {
@@ -72,8 +73,8 @@ func (t *testHolderOperator) ProcessFragment(*fragment) error {
 	return nil
 }
 
-func makeHolder() (*Holder, string, error) {
-	path, err := ioutil.TempDir("", "pilosa-")
+func makeHolder(tb testing.TB) (*Holder, string, error) {
+	path, err := testhook.TempDir(tb, "pilosa-")
 	if err != nil {
 		return nil, "", err
 	}
@@ -109,7 +110,7 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui
 }
 
 func TestHolderOperatorProcess(t *testing.T) {
-	h, path, err := makeHolder()
+	h, path, err := makeHolder(t)
 	if err != nil {
 		t.Fatalf("creating holder: %v", err)
 	}
@@ -139,7 +140,7 @@ func TestHolderOperatorProcess(t *testing.T) {
 }
 
 func TestHolderOperatorCancel(t *testing.T) {
-	h, path, err := makeHolder()
+	h, path, err := makeHolder(t)
 	if err != nil {
 		t.Fatalf("creating holder: %v", err)
 	}

holder_test.go

@@ -33,7 +33,7 @@ import (
 
 func TestHolder_Open(t *testing.T) {
 	t.Run("ErrIndexName", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 
 		bufLogger := test.NewBufferLogger()
 		h.Holder.Logger = bufLogger
@@ -60,7 +60,7 @@ func TestHolder_Open(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
@@ -79,7 +79,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
@@ -99,7 +99,7 @@ func TestHolder_Open(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@@ -119,7 +119,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		var idx *pilosa.Index
@@ -142,7 +142,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		var idx *pilosa.Index
@@ -170,7 +170,7 @@ func TestHolder_Open(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		var idx *pilosa.Index
@@ -205,7 +205,7 @@ func TestHolder_Open(t *testing.T) {
 	t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
 		roaringOnlyTest(t)
 
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		var idx *pilosa.Index
@@ -239,7 +239,7 @@ func TestHolder_Open(t *testing.T) {
 	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
 		roaringOnlyTest(t)
 
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		idx, err := h.CreateIndex("foo", pilosa.IndexOptions{})
@@ -271,7 +271,7 @@ func TestHolder_Open(t *testing.T) {
 
 	t.Run("ForeignIndex", func(t *testing.T) {
 		t.Run("ErrForeignIndexNotFound", func(t *testing.T) {
-			h := test.MustOpenHolder()
+			h := test.MustOpenHolder(t)
 			defer h.Close()
 
 			if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@@ -288,7 +288,7 @@ func TestHolder_Open(t *testing.T) {
 
 		// Foreign index zzz is opened after foo/bar.
 		t.Run("ForeignIndexNotOpenYet", func(t *testing.T) {
-			h := test.MustOpenHolder()
+			h := test.MustOpenHolder(t)
 			defer h.Close()
 
 			if _, err := h.CreateIndex("zzz", pilosa.IndexOptions{}); err != nil {
@@ -308,7 +308,7 @@ func TestHolder_Open(t *testing.T) {
 
 		// Foreign index aaa is opened before foo/bar.
 		t.Run("ForeignIndexIsOpen", func(t *testing.T) {
-			h := test.MustOpenHolder()
+			h := test.MustOpenHolder(t)
 			defer h.Close()
 
 			if _, err := h.CreateIndex("aaa", pilosa.IndexOptions{}); err != nil {
@@ -328,7 +328,7 @@ func TestHolder_Open(t *testing.T) {
 
 		// Try to re-create existing index
 		t.Run("CreateIndexIfNotExists", func(t *testing.T) {
-			h := test.MustOpenHolder()
+			h := test.MustOpenHolder(t)
 			defer h.Close()
 
 			idx1, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
@@ -356,7 +356,7 @@ func TestHolder_Open(t *testing.T) {
 
 func TestHolder_HasData(t *testing.T) {
 	t.Run("IndexDirectory", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if ok, err := h.HasData(); ok || err != nil {
@@ -373,7 +373,7 @@ func TestHolder_HasData(t *testing.T) {
 	})
 
 	t.Run("Peek", func(t *testing.T) {
-		h := test.NewHolder()
+		h := test.NewHolder(t)
 
 		if ok, err := h.HasData(); ok || err != nil {
 			t.Fatal("expected HasData to return false, no err, but", ok, err)
@@ -390,7 +390,7 @@ func TestHolder_HasData(t *testing.T) {
 	})
 
 	t.Run("Peek at missing directory", func(t *testing.T) {
-		h := test.NewHolder()
+		h := test.NewHolder(t)
 
 		// Ensure that hasData is false when dir doesn't exist.
 		h.Path = "bad-path"
@@ -404,7 +404,7 @@ func TestHolder_HasData(t *testing.T) {
 // Ensure holder can delete an index and its underlying files.
 func TestHolder_DeleteIndex(t *testing.T) {
 
-	hldr := test.MustOpenHolder()
+	hldr := test.MustOpenHolder(t)
 	defer hldr.Close()
 
 	// Write bits to separate indexes.
@@ -432,43 +432,43 @@ func TestHolder_DeleteIndex(t *testing.T) {
 // Ensure holder can sync with a remote holder.
 func TestHolderSyncer_SyncHolder(t *testing.T) {
 	c := test.MustNewCluster(t, 2)
-	c[0].Config.Cluster.ReplicaN = 2
-	c[0].Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 2
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.Cluster.ReplicaN = 2
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 2
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
 	}
 	defer c.Close()
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index y: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f0: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field z in y: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool())
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool())
 	if err != nil {
 		t.Fatalf("creating field b in y: %v", err)
 	}
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 	// Set data on the local holder.
 	hldr0.SetBit("i", "f", 0, 10)
@@ -495,11 +495,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 	hldr1.SetBit("y", "b", 0, (3*ShardWidth)+5) // false
 	hldr1.SetBit("y", "b", 1, (3*ShardWidth)+7) // true
 
-	err = c[0].Server.SyncData()
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
-	err = c[1].Server.SyncData()
+	err = c.GetNode(1).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 1: %v", err)
 	}
@@ -543,30 +543,30 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 // the row boundaries of the block.
 func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
 	c := test.MustNewCluster(t, 3)
-	c[0].Config.Cluster.ReplicaN = 3
-	c[0].Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 3
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.Cluster.ReplicaN = 3
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 3
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
 	}
 	defer c.Close()
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
 
 	blockEdge := uint64(pilosa.HashBlockSize)
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
-	hldr2 := &test.Holder{Holder: c[2].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
+	hldr2 := &test.Holder{Holder: c.GetNode(2).Server.Holder()}
 
 	// Set data on the local holder.
 	hldr0.SetBit("i", "f", blockEdge-1, 10)
@@ -579,7 +579,7 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
 
 	// Leave the third replica empty to force a block merge.
 	//
-	err = c[0].Server.SyncData()
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
@@ -598,28 +598,28 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
 // Ensure holder correctly handles clears during block sync.
 func TestHolderSyncer_Clears(t *testing.T) {
 	c := test.MustNewCluster(t, 3)
-	c[0].Config.Cluster.ReplicaN = 3
-	c[0].Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 3
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.Cluster.ReplicaN = 3
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 3
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
 	}
 	defer c.Close()
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
-	hldr2 := &test.Holder{Holder: c[2].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
+	hldr2 := &test.Holder{Holder: c.GetNode(2).Server.Holder()}
 
 	// Set data on the local holder that should be cleared
 	// because it's the only instance of this value.
@@ -631,7 +631,7 @@ func TestHolderSyncer_Clears(t *testing.T) {
 	hldr1.SetBit("i", "f", 0, 20)
 	hldr2.SetBit("i", "f", 0, 20)
 
-	err = c[0].Server.SyncData()
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
@@ -647,10 +647,10 @@ func TestHolderSyncer_Clears(t *testing.T) {
 // Ensure holder can sync time quantum views with a remote holder.
 func TestHolderSyncer_TimeQuantum(t *testing.T) {
 	c := test.MustNewCluster(t, 2)
-	c[0].Config.Cluster.ReplicaN = 2
-	c[0].Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 2
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.Cluster.ReplicaN = 2
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 2
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
@@ -659,17 +659,17 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 
 	quantum := "D"
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum)))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum)))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 	// Set data on the local holder for node0.
 	t1 := time.Date(2018, 8, 1, 12, 30, 0, 0, time.UTC)
@@ -680,7 +680,7 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 	// Set data on node1.
 	hldr1.SetBitTime("i", "f", 0, 22, &t2)
 
-	err = c[0].Server.SyncData()
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
@@ -700,10 +700,10 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 func TestHolderSyncer_IntField(t *testing.T) {
 	t.Run("BasicSync", func(t *testing.T) {
 		c := test.MustNewCluster(t, 2)
-		c[0].Config.Cluster.ReplicaN = 2
-		c[0].Config.AntiEntropy.Interval = 0
-		c[1].Config.Cluster.ReplicaN = 2
-		c[1].Config.AntiEntropy.Interval = 0
+		c.GetNode(0).Config.Cluster.ReplicaN = 2
+		c.GetNode(0).Config.AntiEntropy.Interval = 0
+		c.GetNode(1).Config.Cluster.ReplicaN = 2
+		c.GetNode(1).Config.AntiEntropy.Interval = 0
 		err := c.Start()
 		if err != nil {
 			t.Fatalf("starting cluster: %v", err)
@@ -712,18 +712,18 @@ func TestHolderSyncer_IntField(t *testing.T) {
 
 		var idx0 *pilosa.Index
 		_ = idx0
-		idx0, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 		_ = idx0
 		if err != nil {
 			t.Fatalf("creating index i: %v", err)
 		}
-		_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100))
+		_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100))
 		if err != nil {
 			t.Fatalf("creating field f: %v", err)
 		}
 
-		hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-		hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+		hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+		hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 		// Set data on the local holder for node0. columnID=1, value=1
 		hldr0.SetValue("i", "f", 1, 1)
@@ -734,7 +734,7 @@ func TestHolderSyncer_IntField(t *testing.T) {
 		idx1 := hldr1.SetValue("i", "f", 2, 2)
 		_ = idx1
 
-		err = c[0].Server.SyncData()
+		err = c.GetNode(0).Server.SyncData()
 		if err != nil {
 			t.Fatalf("syncing node 0: %v", err)
 		}
@@ -758,10 +758,10 @@ func TestHolderSyncer_IntField(t *testing.T) {
 
 	t.Run("MultiShard", func(t *testing.T) {
 		c := test.MustNewCluster(t, 2)
-		c[0].Config.Cluster.ReplicaN = 2
-		c[0].Config.AntiEntropy.Interval = 0
-		c[1].Config.Cluster.ReplicaN = 2
-		c[1].Config.AntiEntropy.Interval = 0
+		c.GetNode(0).Config.Cluster.ReplicaN = 2
+		c.GetNode(0).Config.AntiEntropy.Interval = 0
+		c.GetNode(1).Config.Cluster.ReplicaN = 2
+		c.GetNode(1).Config.AntiEntropy.Interval = 0
 		err := c.Start()
 		if err != nil {
 			t.Fatalf("starting cluster: %v", err)
@@ -770,18 +770,18 @@ func TestHolderSyncer_IntField(t *testing.T) {
 
 		var idx0 *pilosa.Index
 		_ = idx0
-		idx0, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 		_ = idx0
 		if err != nil {
 			t.Fatalf("creating index i: %v", err)
 		}
-		_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
 		if err != nil {
 			t.Fatalf("creating field f: %v", err)
 		}
 
-		hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-		hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+		hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+		hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 		// Set data on the local holder for node0.
 		hldr0.SetValue("i", "f", 1*pilosa.ShardWidth, 11)
@@ -799,11 +799,11 @@ func TestHolderSyncer_IntField(t *testing.T) {
 		// node0: [0,3,7]
 		// node1: [1,2,4]
 
-		err = c[0].Server.SyncData()
+		err = c.GetNode(0).Server.SyncData()
 		if err != nil {
 			t.Fatalf("syncing node 0: %v", err)
 		}
-		err = c[1].Server.SyncData()
+		err = c.GetNode(1).Server.SyncData()
 		if err != nil {
 			t.Fatalf("syncing node 1: %v", err)
 		}

http/client_test.go

@@ -49,7 +49,7 @@ func TestClient_MultiNode(t *testing.T) {
 	defer c.Close()
 
 	hldr := []test.Holder{}
-	for _, command := range c {
+	for _, command := range c.Nodes {
 		hldr = append(hldr, test.Holder{Holder: command.Server.Holder()})
 	}
 
@@ -72,7 +72,7 @@ func TestClient_MultiNode(t *testing.T) {
 			}
 		}
 		if !ownsNum {
-			t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c[i].URL(), owns)
+			t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c.GetNode(i).URL(), owns)
 		}
 	}
 
@@ -86,11 +86,11 @@ func TestClient_MultiNode(t *testing.T) {
 			maxShard = x
 		}
 	}
-	_, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err := c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
 	if err != nil {
 		t.Fatalf("creating field: %v", err)
 	}
@@ -119,24 +119,24 @@ func TestClient_MultiNode(t *testing.T) {
 	// Rebuild the RankCache.
 	// We have to do this to avoid the 10-second cache invalidation delay
 	// built into cache.Invalidate()
-	err = c[0].RecalculateCaches(t)
+	err = c.GetNode(0).RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
-	err = c[1].RecalculateCaches(t)
+	err = c.GetNode(1).RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
-	err = c[2].RecalculateCaches(t)
+	err = c.GetNode(2).RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
 
 	// Connect to each node to compare results.
 	client := make([]*Client, 3)
-	client[0] = MustNewClient(c[0].URL(), http.GetHTTPClient(nil))
-	client[1] = MustNewClient(c[1].URL(), http.GetHTTPClient(nil))
-	client[2] = MustNewClient(c[2].URL(), http.GetHTTPClient(nil))
+	client[0] = MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil))
+	client[1] = MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil))
+	client[2] = MustNewClient(c.GetNode(2).URL(), http.GetHTTPClient(nil))
 
 	topN := 4
 	queryRequest := &pilosa.QueryRequest{
@@ -188,7 +188,7 @@ func TestClient_MultiNode(t *testing.T) {
 func TestClient_Export(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	host := cmd.URL()
 
@@ -368,7 +368,7 @@ func TestClient_Export(t *testing.T) {
 func TestClient_Import(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	host := cmd.URL()
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -415,7 +415,7 @@ func TestClient_Import(t *testing.T) {
 // Ensure client can bulk import column attrs.
 func TestClient_ImportColumnAttrs(t *testing.T) {
 	cluster := test.MustNewCluster(t, 2)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -425,31 +425,31 @@ func TestClient_ImportColumnAttrs(t *testing.T) {
 	defer cluster.Close()
 
 	ctx := context.Background()
-	_, err = cluster[0].API.CreateIndex(ctx, "i", pilosa.IndexOptions{})
+	_, err = cluster.GetNode(0).API.CreateIndex(ctx, "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
-	_, err = cluster[0].API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
+	_, err = cluster.GetNode(0).API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
 	if err != nil {
 		t.Fatalf("creating field: %v", err)
 	}
-	_, err = cluster[0].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"})
+	_, err = cluster.GetNode(0).API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"})
 	if err != nil {
 		t.Fatalf("querying: %v", err)
 	}
 
 	attrKey := "k"
 	// Send import request.
-	host := cluster[0].URL()
+	host := cluster.GetNode(0).URL()
 	c := MustNewClient(host, http.GetHTTPClient(nil))
 	colAttrsReq := makeImportColumnAttrsRequest("i", 0, attrKey)
-	if err := c.ImportColumnAttrs(ctx, &cluster[1].API.Node().URI, "i", colAttrsReq); err != nil {
+	if err := c.ImportColumnAttrs(ctx, &cluster.GetNode(1).API.Node().URI, "i", colAttrsReq); err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify data.
 	pql := "Options(Row(f=0), columnAttrs=true)"
-	res, err := cluster[1].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql})
+	res, err := cluster.GetNode(1).API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -469,7 +469,7 @@ func TestClient_ImportColumnAttrs(t *testing.T) {
 // Ensure client can bulk import data.
 func TestClient_ImportRoaring(t *testing.T) {
 	cluster := test.MustNewCluster(t, 2)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -478,29 +478,29 @@ func TestClient_ImportRoaring(t *testing.T) {
 	}
 	defer cluster.Close()
 
-	_, err = cluster[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = cluster.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
-	_, err = cluster[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
+	_, err = cluster.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
 	if err != nil {
 		t.Fatalf("creating field: %v", err)
 	}
-	_, err = cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"})
+	_, err = cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"})
 	if err != nil {
 		t.Fatalf("querying: %v", err)
 	}
 
 	// Send import request.
-	host := cluster[0].URL()
+	host := cluster.GetNode(0).URL()
 	c := MustNewClient(host, http.GetHTTPClient(nil))
 	// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
 	roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100")
-	if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
 		t.Fatal(err)
 	}
 
-	hldr := test.Holder{Holder: cluster[0].Server.Holder()}
+	hldr := test.Holder{Holder: cluster.GetNode(0).Server.Holder()}
 	// Verify data on node 0.
 	if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
 		t.Fatalf("unexpected columns: %+v", a)
@@ -509,7 +509,7 @@ func TestClient_ImportRoaring(t *testing.T) {
 		t.Fatalf("unexpected columns: %+v", a)
 	}
 
-	hldr2 := test.Holder{Holder: cluster[1].Server.Holder()}
+	hldr2 := test.Holder{Holder: cluster.GetNode(1).Server.Holder()}
 	// Verify data on node 1.
 	if a := hldr2.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
 		t.Fatalf("unexpected columns: %+v", a)
@@ -521,7 +521,7 @@ func TestClient_ImportRoaring(t *testing.T) {
 	// Ensure that sending a roaring import with the clear flag works as expected.
 	// [65539, 65540]
 	roaringReq = makeImportRoaringRequest(true, "3A30000001000000010001001000000003000400")
-	if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
 		t.Fatal(err)
 	}
 
@@ -544,7 +544,7 @@ func TestClient_ImportRoaring(t *testing.T) {
 	// Ensure that sending a roaring import with the clear flag works as expected.
 	// [4, 6, 65537, 65539]
 	roaringReq = makeImportRoaringRequest(true, "3A300000020000000000010001000100180000001C0000000400060001000300")
-	if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
 		t.Fatal(err)
 	}
 
@@ -567,7 +567,7 @@ func TestClient_ImportRoaring(t *testing.T) {
 	// Ensure that sending a roaring import with the clear flag works as expected.
 	// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
 	roaringReq = makeImportRoaringRequest(true, "3B3001000100000900010000000100010009000100")
-	if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
 		t.Fatal(err)
 	}
 
@@ -593,7 +593,7 @@ func TestClient_ImportKeys(t *testing.T) {
 	t.Run("SingleNode", func(t *testing.T) {
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		host := cmd.URL()
 
 		cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true})
@@ -691,8 +691,8 @@ func TestClient_ImportKeys(t *testing.T) {
 	t.Run("MultiNode", func(t *testing.T) {
 		cluster := test.MustRunCluster(t, 2)
 		defer cluster.Close()
-		cmd0 := cluster[0]
-		cmd1 := cluster[1]
+		cmd0 := cluster.GetNode(0)
+		cmd1 := cluster.GetNode(1)
 		host0 := cmd0.URL()
 		host1 := cmd1.URL()
 
@@ -768,7 +768,7 @@ func TestClient_ImportKeys(t *testing.T) {
 	t.Run("IntegerFieldSingleNode", func(t *testing.T) {
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		host := cmd.URL()
 		holder := cmd.Server.Holder()
 		hldr := test.Holder{Holder: holder}
@@ -840,7 +840,7 @@ func TestClient_ImportIDs(t *testing.T) {
 	t.Run("ImportRangeImport", func(t *testing.T) {
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		host := cmd.URL()
 		holder := cmd.Server.Holder()
 		hldr := test.Holder{Holder: holder}
@@ -901,7 +901,7 @@ func TestClient_ImportIDs(t *testing.T) {
 func TestClient_ImportValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	host := cmd.URL()
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -974,7 +974,7 @@ func TestClient_ImportValue(t *testing.T) {
 func TestClient_ImportExistence(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	host := cmd.URL()
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -1053,7 +1053,7 @@ func TestClient_ImportExistence(t *testing.T) {
 func TestClient_FragmentBlocks(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -1086,7 +1086,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
 func TestClient_CreateDecimalField(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil))
 
@@ -1195,8 +1195,8 @@ func TestClientTransactions(t *testing.T) {
 	c := test.MustRunCluster(t, 3)
 	defer c.Close()
 
-	client0 := MustNewClient(c[0].URL(), http.GetHTTPClient(nil))
-	client1 := MustNewClient(c[1].URL(), http.GetHTTPClient(nil))
+	client0 := MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil))
+	client1 := MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil))
 
 	// can create, list, get, and finish a transaction
 	var expDeadline time.Time

http/translator_test.go

@@ -36,7 +36,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 			cluster := test.MustRunCluster(t, 1)
 			defer cluster.Close()
-			primary := cluster[0]
+			primary := cluster.GetNode(0)
 
 			hldr := test.Holder{Holder: primary.Server.Holder()}
 			index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
@@ -157,7 +157,7 @@ func benchmarkSetup(b *testing.B, ctx context.Context, key string, nkeys int) (s
 	b.Helper()
 
 	cluster := test.MustRunCluster(b, 1)
-	primary := cluster[0]
+	primary := cluster.GetNode(0)
 
 	idx := primary.MustCreateIndex(b, "i", pilosa.IndexOptions{})
 	fld := primary.MustCreateField(b, idx.Name(), "f", pilosa.OptFieldKeys())

index.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pilosa/pilosa/v2/internal"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
@@ -153,6 +154,9 @@ func (i *Index) CreatedAt() int64 {
 // Name returns name of the index.
 func (i *Index) Name() string { return i.name }
 
+// Holder yields this index's Holder.
+func (i *Index) Holder() *Holder { return i.holder }
+
 // QualifiedName returns the qualified name of the index.
 func (i *Index) QualifiedName() string { return i.qualifiedName }
 
@@ -253,6 +257,7 @@ func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) {
 		return err
 	}
 
+	_ = testhook.Opened(i.holder.Auditor, i, nil)
 	return nil
 }
 
@@ -339,7 +344,16 @@ fileLoop:
 			})
 		}
 	}
-	return eg.Wait()
+	err = eg.Wait()
+	if err != nil {
+		// Close any fields which got opened, since the overall
+		// index won't be open.
+		for n, f := range i.fields {
+			f.Close()
+			delete(i.fields, n)
+		}
+	}
+	return err
 }
 
 // openExistenceField gets or creates the existence field and associates it to the index.
@@ -403,6 +417,9 @@ func (i *Index) saveMeta() error {
 func (i *Index) Close() error {
 	i.mu.Lock()
 	defer i.mu.Unlock()
+	defer func() {
+		_ = testhook.Closed(i.holder.Auditor, i, nil)
+	}()
 
 	err := i.Txf.CloseIndex(i)
 	if err != nil {

index_internal_test.go

@@ -15,19 +15,23 @@
 package pilosa
 
 import (
-	"io/ioutil"
 	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // mustOpenIndex returns a new, opened index at a temporary path. Panic on error.
-func mustOpenIndex(opt IndexOptions) *Index {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
+func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}
 	h := NewHolder(1)
 	h.Path = path
 	index, err := h.CreateIndex("i", opt)
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 
 	if err != nil {
 		panic(err)
@@ -36,9 +40,6 @@ func mustOpenIndex(opt IndexOptions) *Index {
 	index.keys = opt.Keys
 	index.trackExistence = opt.TrackExistence
 
-	if err := index.Open(false); err != nil {
-		panic(err)
-	}
 	return index
 }
 
@@ -56,7 +57,7 @@ func (i *Index) reopen() error {
 // Ensure that deleting the existence field is handled properly.
 func TestIndex_Existence_Delete(t *testing.T) {
 	// Create Index (with existence tracking).
-	index := mustOpenIndex(IndexOptions{TrackExistence: true})
+	index := mustOpenIndex(t, IndexOptions{TrackExistence: true})
 	defer index.Close()
 
 	// Ensure existence field has been created.

index_test.go

@@ -15,13 +15,13 @@
 package pilosa_test
 
 import (
-	"io/ioutil"
 	"reflect"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -30,7 +30,7 @@ const ShardWidth = pilosa.ShardWidth
 
 // Ensure index can open and retrieve a field.
 func TestIndex_CreateFieldIfNotExists(t *testing.T) {
-	index := test.MustOpenIndex()
+	index := test.MustOpenIndex(t)
 	defer index.Close()
 
 	// Create field.
@@ -58,7 +58,7 @@ func TestIndex_CreateField(t *testing.T) {
 	// Ensure time quantum can be set appropriately on a new field.
 	t.Run("TimeQuantum", func(t *testing.T) {
 		t.Run("Explicit", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			// Create field with explicit quantum.
@@ -74,7 +74,7 @@ func TestIndex_CreateField(t *testing.T) {
 	// Ensure time quantum can be set appropriately on a new field.
 	t.Run("TimeQuantumNoStandardView", func(t *testing.T) {
 		t.Run("Explicit", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			// Create field with explicit quantum with no standard view
@@ -90,7 +90,7 @@ func TestIndex_CreateField(t *testing.T) {
 	// Ensure field can include range columns.
 	t.Run("BSIFields", func(t *testing.T) {
 		t.Run("OK", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			// Create field with schema and verify it exists.
@@ -112,7 +112,7 @@ func TestIndex_CreateField(t *testing.T) {
 		// on field creation FieldOptions validation.
 		/*
 			t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -123,7 +123,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
 					CacheType: pilosa.CacheTypeNone,
@@ -134,7 +134,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("ErrFieldFieldsAllowed", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -147,7 +147,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("ErrFieldNameRequired", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -160,7 +160,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("ErrInvalidFieldType", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -173,7 +173,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("ErrInvalidBSIGroupRange", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -190,7 +190,7 @@ func TestIndex_CreateField(t *testing.T) {
 	t.Run("WithKeys", func(t *testing.T) {
 		// Don't allow an int field to be created with keys=true
 		t.Run("IntField", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			_, err := index.CreateField("f", pilosa.OptFieldTypeInt(-1, 1), pilosa.OptFieldKeys())
@@ -201,7 +201,7 @@ func TestIndex_CreateField(t *testing.T) {
 
 		// Don't allow a decimal field to be created with keys=true
 		t.Run("DecimalField", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			_, err := index.CreateField("f", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -1}, pql.Decimal{Value: 1}), pilosa.OptFieldKeys())
@@ -214,7 +214,7 @@ func TestIndex_CreateField(t *testing.T) {
 
 // Ensure index can delete a field.
 func TestIndex_DeleteField(t *testing.T) {
-	index := test.MustOpenIndex()
+	index := test.MustOpenIndex(t)
 	defer index.Close()
 
 	// Create field.
@@ -238,7 +238,7 @@ func TestIndex_DeleteField(t *testing.T) {
 
 // Ensure index can validate its name.
 func TestIndex_InvalidName(t *testing.T) {
-	path, err := ioutil.TempDir("", "pilosa-index-")
+	path, err := testhook.TempDir(t, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}

lmdb.go

@@ -22,9 +22,11 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -128,14 +130,15 @@ func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
 
 	err = env.SetMaxDBs(1)
 	panicOn(err)
-	err = env.SetMapSize(256 << 30) // 256GB
+	//err = env.SetMapSize(256 << 30) // 256GB
+	err = env.SetMapSize(16 << 30) // 16GB
 	panicOn(err)
 
 	panicOn(os.MkdirAll(filepath.Dir(path), 0755))
 
 	flags := uint(lmdb.NoReadahead | lmdb.NoSubdir)
 
-	// unsafe, but get upper bound on performance. TODO: remove these.
+	// unsafe, but get upper bound on performance.
 	//	WriteMap    = C.MDB_WRITEMAP   // Use a writable memory map.
 	//	NoMetaSync  = C.MDB_NOMETASYNC // Don't fsync metapage after commit.
 	//	NoSync      = C.MDB_NOSYNC     // Don't fsync after commit.
@@ -145,12 +148,20 @@ func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
 	// kRemove  N=  710401   avg/op:      7.714µs   sd:      27.83µs  total: 5.480656859s
 	//    kAdd  N=  722835   avg/op:      9.096µs   sd:    105.787µs  total: 6.575497725s
 
+	// ACI not ACID at the moment; no durability
 	flags = flags |
+		lmdb.NoMemInit | // Disable LMDB memory initialization
+
+		// Note that lmdb.WriteMap requests a big, writable, memory map.
+		// On my darwin/OSX laptop with 16GB ram, for instance, we
+		// can have difficulty obtaining this, resulting in
+		//   panic: mdb_env_open: no space left on device
 		lmdb.WriteMap | // Use a writable memory map.
-		//lmdb.NoMetaSync | // Don't fsync metapage after commit.
-		//lmdb.NoSync | // Don't fsync after commit.
-		//lmdb.MapAsync | // Flush asynchronously when using the WriteMap flag.
-		lmdb.NoMemInit // Disable LMDB memory initialization
+
+		// default ACI (not Durable) transactions; 300% faster write speed results.
+		lmdb.NoMetaSync | // Don't fsync metapage after commit.
+		lmdb.NoSync | // Don't fsync after commit.
+		lmdb.MapAsync // Flush asynchronously when using the WriteMap flag.
 
 	err = env.Open(path, flags, 0644)
 	if err != nil {
@@ -476,80 +487,86 @@ func (tx *LMDBTx) RemoveContainer(index, field, view string, shard uint64, ckey
 
 // Add sets all the a bits hot in the specified fragment.
 func (tx *LMDBTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
+	return tx.addOrRemove(index, field, view, shard, batched, false, a...)
+}
 
+// Remove clears all the specified a bits in the chosen fragment.
+func (tx *LMDBTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+	const batched = false
+	const remove = true
+	return tx.addOrRemove(index, field, view, shard, batched, remove, a...)
+}
+
+func (tx *LMDBTx) addOrRemove(index, field, view string, shard uint64, batched, remove bool, a ...uint64) (changeCount int, err error) {
 	// pure hack to match RoaringTx
 	defer func() {
-		if !batched {
+		if !remove && !batched {
 			if changeCount > 0 {
 				changeCount = 1
 			}
 		}
 	}()
 
-	// TODO: optimization: group 'a' elements into their containers,
-	// and then do all the Adds on that
-	// container at once, so we don't retrieve a container per bit.
-	// (maybe, for example, using ImportRoaringBits with clear=false).
-
-	for _, v := range a {
-		hi, lo := highbits(v), lowbits(v)
-
-		var rct *roaring.Container
-		rct, err = tx.Container(index, field, view, shard, hi)
-		panicOn(err)
-		if err != nil {
-			return 0, err
-		}
-		chng := false
-		// TODO optimization: set all the bits in the current container at once. group by container first.
-		rc1, chng := rct.Add(lo)
-		panicOn(err)
-		if chng {
-			changeCount++
-		}
-		if err != nil {
-			return changeCount, err
-		}
-		err = tx.PutContainer(index, field, view, shard, hi, rc1)
-		//panicOn(err)
+	if len(a) == 0 {
+		return 0, nil
 	}
-	return
-}
 
-// Remove clears all the specified a bits in the chosen fragment.
-func (tx *LMDBTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
+	// have to sort, b/c input is not always sorted.
+	sort.Slice(a, func(i, j int) bool { return a[i] < a[j] })
 
-	// TODO: optimization: group 'a' elements into their containers,
-	// and then do all the Removes on that
-	// container at once, so we don't retrieve a container per bit.
-	// (maybe, for example, using ImportRoaringBits with clear=true).
-	for _, v := range a {
-		hi, lo := highbits(v), lowbits(v)
+	var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter.
+	var rc *roaring.Container
+	var hi uint64
+	var lo uint16
 
-		var rct *roaring.Container
-		rct, err = tx.Container(index, field, view, shard, hi)
-		panicOn(err)
-		if err != nil {
-			return 0, err
-		}
+	for i, v := range a {
+
+		hi, lo = highbits(v), lowbits(v)
+		if hi != lastHi {
+			// either first time through, or changed to a different container.
+			// do we need put the last updated container now?
+			if i > 0 {
+				// not first time through, write what we got.
+				if remove && (rc == nil || rc.N() == 0) {
+					err = tx.RemoveContainer(index, field, view, shard, lastHi)
+					panicOn(err)
+				} else {
+					err = tx.PutContainer(index, field, view, shard, lastHi, rc)
+					panicOn(err)
+				}
+			}
+			// get the next container
+			rc, err = tx.Container(index, field, view, shard, hi)
+			panicOn(err)
+		} // else same container, keep adding bits to rct.
 		chng := false
-		rc1, chng := rct.Remove(lo)
-		panicOn(err)
+		// rc can be nil before, and nil after, in both Remove/Add below.
+		// The roaring container add() and remove() methods handle this.
+		if remove {
+			rc, chng = rc.Remove(lo)
+		} else {
+			rc, chng = rc.Add(lo)
+		}
 		if chng {
 			changeCount++
 		}
-		if err != nil {
-			return changeCount, err
-		}
-		if rc1.N() == 0 {
+		lastHi = hi
+	}
+	// write the last updates.
+	if remove {
+		if rc == nil || rc.N() == 0 {
 			err = tx.RemoveContainer(index, field, view, shard, hi)
-			if err != nil {
-				return
-			}
+			panicOn(err)
 		} else {
-			err = tx.PutContainer(index, field, view, shard, hi, rc1)
+			err = tx.PutContainer(index, field, view, shard, hi, rc)
 			panicOn(err)
 		}
+	} else {
+		if rc == nil || rc.N() == 0 {
+			panic("there should be no way to have an empty bitmap AFTER an Add() operation")
+		}
+		err = tx.PutContainer(index, field, view, shard, hi, rc)
+		panicOn(err)
 	}
 	return
 }

main_test.go

@@ -0,0 +1,25 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+func TestMain(m *testing.M) {
+	testhook.RunTestsWithHooks(m)
+}

mmap_test.go

@@ -31,11 +31,12 @@ type cv struct {
 
 func forceSnapshotsCheckMapping(t *testing.T) {
 	depth := uint(6)
-	f, idx := mustOpenBSIFragment("i", "f", viewStandard, 0)
+	f, idx, tx := mustOpenBSIFragment(t, "i", "f", viewStandard, 0)
+	tx.Rollback()
 	f.Logger = logger.NewLogfLogger(t)
 	defer f.Clean(t)
 
-	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
+	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
 	defer tx.Rollback()
 
 	for i := 0; i < f.MaxOpN; i++ {

pg/server_test.go

@@ -35,7 +35,7 @@ func TestStartupTimeout(t *testing.T) {
 
 	connect, shutdown, err := pgtest.ServeMem(&pg.Server{
 		StartupTimeout: time.Millisecond,
-		Logger:         logger.NewLogfLogger(t),
+		Logger:         logger.NopLogger,
 	})
 	if err != nil {
 		t.Fatalf("starting in-memory postgres server: %v", err)
@@ -103,7 +103,7 @@ func TestPQConnect(t *testing.T) {
 
 	server := &pg.Server{
 		StartupTimeout: time.Second,
-		Logger:         logger.NewLogfLogger(t),
+		Logger:         logger.NopLogger,
 	}
 	addr, shutdown, err := pgtest.ServeTCP(":0", server)
 	if err != nil {
@@ -134,7 +134,7 @@ func TestPQConnectSSL(t *testing.T) {
 
 	server := &pg.Server{
 		StartupTimeout: time.Second,
-		Logger:         logger.NewLogfLogger(t),
+		Logger:         logger.NopLogger,
 	}
 	addr, shutdown, err := pgtest.ServeTLS(":0", server)
 	if err != nil {
@@ -197,7 +197,7 @@ func TestPSQLQuery(t *testing.T) {
 		}),
 		TypeEngine:     pg.PrimitiveTypeEngine{},
 		StartupTimeout: time.Second,
-		Logger:         logger.NewLogfLogger(t),
+		Logger:         logger.NopLogger,
 	}
 	addr, shutdown, err := pgtest.ServeTCP(":0", server)
 	if err != nil {

rbf.go

@@ -19,7 +19,9 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math"
 	"os"
+	"sort"
 	"strings"
 	"sync"
 
@@ -164,12 +166,90 @@ func (tx *RBFTx) RemoveContainer(index, field, view string, shard uint64, key ui
 	return tx.tx.RemoveContainer(rbfName(index, field, view, shard), key)
 }
 
+// Add sets all the a bits hot in the specified fragment.
 func (tx *RBFTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
-	return tx.tx.Add(rbfName(index, field, view, shard), a...)
+	return tx.addOrRemove(index, field, view, shard, batched, false, a...)
 }
 
+// Remove clears all the specified a bits in the chosen fragment.
 func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
-	return tx.tx.Remove(rbfName(index, field, view, shard), a...)
+	const batched = false
+	const remove = true
+	return tx.addOrRemove(index, field, view, shard, batched, remove, a...)
+}
+
+func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, batched, remove bool, a ...uint64) (changeCount int, err error) {
+	// pure hack to match RoaringTx
+	defer func() {
+		if !remove && !batched {
+			if changeCount > 0 {
+				changeCount = 1
+			}
+		}
+	}()
+
+	if len(a) == 0 {
+		return 0, nil
+	}
+
+	// have to sort, b/c input is not always sorted.
+	sort.Slice(a, func(i, j int) bool { return a[i] < a[j] })
+
+	var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter.
+	var rc *roaring.Container
+	var hi uint64
+	var lo uint16
+
+	for i, v := range a {
+
+		hi, lo = highbits(v), lowbits(v)
+		if hi != lastHi {
+			// either first time through, or changed to a different container.
+			// do we need put the last updated container now?
+			if i > 0 {
+				// not first time through, write what we got.
+				if remove && (rc == nil || rc.N() == 0) {
+					err = tx.RemoveContainer(index, field, view, shard, lastHi)
+					panicOn(err)
+				} else {
+					err = tx.PutContainer(index, field, view, shard, lastHi, rc)
+					panicOn(err)
+				}
+			}
+			// get the next container
+			rc, err = tx.Container(index, field, view, shard, hi)
+			panicOn(err)
+		} // else same container, keep adding bits to rct.
+		chng := false
+		// rc can be nil before, and nil after, in both Remove/Add below.
+		// The roaring container add() and remove() methods handle this.
+		if remove {
+			rc, chng = rc.Remove(lo)
+		} else {
+			rc, chng = rc.Add(lo)
+		}
+		if chng {
+			changeCount++
+		}
+		lastHi = hi
+	}
+	// write the last updates.
+	if remove {
+		if rc == nil || rc.N() == 0 {
+			err = tx.RemoveContainer(index, field, view, shard, hi)
+			panicOn(err)
+		} else {
+			err = tx.PutContainer(index, field, view, shard, hi, rc)
+			panicOn(err)
+		}
+	} else {
+		if rc == nil || rc.N() == 0 {
+			panic("there should be no way to have an empty bitmap AFTER an Add() operation")
+		}
+		err = tx.PutContainer(index, field, view, shard, hi, rc)
+		panicOn(err)
+	}
+	return
 }
 
 func (tx *RBFTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {

rbf/tx_test.go

@@ -94,7 +94,7 @@ func TestTx_CommitRollback(t *testing.T) {
 	})
 
 	t.Run("SingleWriter", func(t *testing.T) {
-		t.Skip("NEED TO FIX IN RACE") //TODO (twg)
+		//t.Skip("NEED TO FIX IN RACE") //TODO (twg)
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
 

rrtx.go

@@ -369,6 +369,7 @@ func (db *RoaringStore) DeleteField(index, field, fieldPath string) error {
 }
 
 // frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
+// The fragment should be closed before this.
 func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
 
 	fragment, ok := frag.(*fragment)
@@ -376,11 +377,6 @@ func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64,
 		return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
 	}
 
-	// Close data files before deletion.
-	if err := fragment.Close(); err != nil {
-		return errors.Wrap(err, "closing fragment")
-	}
-
 	// Delete fragment file.
 	if err := os.Remove(fragment.path); err != nil {
 		return errors.Wrap(err, "deleting fragment file")

server/cluster_test.go

@@ -35,7 +35,7 @@ import (
 // Ensure program can send/receive broadcast messages.
 func TestMain_SendReceiveMessage(t *testing.T) {
 	ms := test.MustRunCluster(t, 2)
-	m0, m1 := ms[0], ms[1]
+	m0, m1 := ms.GetNode(0), ms.GetNode(1)
 	defer ms.Close()
 
 	// Expected indexes and Fields
@@ -131,10 +131,10 @@ func TestClusterResize_EmptyNodes(t *testing.T) {
 	clus := test.MustRunCluster(t, 2)
 	defer clus.Close()
 
-	if clus[0].API.State() != pilosa.ClusterStateNormal {
-		t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State())
-	} else if clus[1].API.State() != pilosa.ClusterStateNormal {
-		t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State())
+	if clus.GetNode(0).API.State() != pilosa.ClusterStateNormal {
+		t.Fatalf("unexpected node0 cluster state: %s", clus.GetNode(0).API.State())
+	} else if clus.GetNode(1).API.State() != pilosa.ClusterStateNormal {
+		t.Fatalf("unexpected node1 cluster state: %s", clus.GetNode(1).API.State())
 	}
 }
 
@@ -144,15 +144,15 @@ func TestClusterResize_AddNode(t *testing.T) {
 		clus := test.MustRunCluster(t, 2)
 		defer clus.Close()
 
-		if !test.CheckClusterState(clus[0], pilosa.ClusterStateNormal, 1000) {
-			t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State())
-		} else if !test.CheckClusterState(clus[1], pilosa.ClusterStateNormal, 1000) {
-			t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State())
+		if !test.CheckClusterState(clus.GetNode(0), pilosa.ClusterStateNormal, 1000) {
+			t.Fatalf("unexpected node0 cluster state: %s", clus.GetNode(0).API.State())
+		} else if !test.CheckClusterState(clus.GetNode(1), pilosa.ClusterStateNormal, 1000) {
+			t.Fatalf("unexpected node1 cluster state: %s", clus.GetNode(1).API.State())
 		}
 	})
 	t.Run("WithIndex", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -168,7 +168,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		}
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -198,7 +198,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		skipTestUnderBlueGreenWithRoaring(t)
 
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -229,7 +229,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -250,7 +250,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 	})
 	t.Run("OneShard", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -278,7 +278,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -299,7 +299,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 	})
 	t.Run("SkippedShard", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -331,7 +331,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -356,7 +356,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 	t.Run("WithIndex", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -378,7 +378,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		}()
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -399,7 +399,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 	})
 	t.Run("ContinuousShards", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -431,7 +431,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -457,7 +457,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 	})
 	t.Run("SkippedShard", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -489,7 +489,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		errc := make(chan error, 1)
@@ -515,7 +515,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 	})
 	t.Run("WithIndexKeys", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -545,7 +545,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		errc := make(chan error, 1)
@@ -573,7 +573,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 func TestCluster_GossipMembership(t *testing.T) {
 	t.Run("Node0Down", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -581,7 +581,7 @@ func TestCluster_GossipMembership(t *testing.T) {
 		var eg errgroup.Group
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		defer m1.Close()
 		eg.Go(func() error {
 			m1.Config.Gossip.Port = "0"
@@ -595,7 +595,7 @@ func TestCluster_GossipMembership(t *testing.T) {
 		})
 
 		// Configure node1
-		m2 := test.NewCommandNode(false)
+		m2 := test.NewCommandNode(t, false)
 		defer m2.Close()
 		eg.Go(func() error {
 			m2.Config.Gossip.Port = "0"
@@ -630,8 +630,8 @@ func TestCluster_GossipMembership(t *testing.T) {
 func TestClusterResize_RemoveNode(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
-	m0 := cluster[0]
-	m1 := cluster[1]
+	m0 := cluster.GetNode(0)
+	m1 := cluster.GetNode(1)
 
 	mustNodeID := func(baseURL string) string {
 		body := test.Do(t, "GET", fmt.Sprintf("%s/status", baseURL), "").Body
@@ -730,7 +730,7 @@ func TestClusterMutualTLS(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 3, commandOpts...)
 	defer cluster.Close()
-	m0 := cluster[0]
+	m0 := cluster.GetNode(0)
 
 	client0 := m0.Client()
 	if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {

server/grpc.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"net"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pilosa/pilosa/v2"
@@ -909,6 +910,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
 
 type grpcServer struct {
 	api        *pilosa.API
+	mu         sync.Mutex
 	grpcServer *grpc.Server
 	ln         net.Listener
 
@@ -956,11 +958,13 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error {
 	}
 
 	// create grpc server
+	s.mu.Lock()
 	s.grpcServer = grpc.NewServer(opts...)
 	pb.RegisterPilosaServer(s.grpcServer, NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats))
 
 	// register the server so its services are available to grpc_cli and others
 	reflection.Register(s.grpcServer)
+	s.mu.Unlock()
 
 	// and start...
 	if err := s.grpcServer.Serve(s.ln); err != nil {
@@ -969,6 +973,16 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error {
 	return nil
 }
 
+// Stop stops the GRPC server. There's no error because the underlying GRPC
+// stuff doesn't report an error.
+func (s *grpcServer) Stop() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.grpcServer != nil {
+		s.grpcServer.Stop()
+	}
+}
+
 func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {
 	server := &grpcServer{
 		logger: logger.NopLogger,

server/grpc_test.go

@@ -709,6 +709,7 @@ func TestQuerySQLUnary(t *testing.T) {
 					{"Table", "string"},
 				},
 				rows: []row{
+					{[]columnResponse{"delete_me"}},
 					{[]columnResponse{"grouper"}},
 					{[]columnResponse{"joiner"}},
 				},
@@ -731,6 +732,27 @@ func TestQuerySQLUnary(t *testing.T) {
 			},
 			eq: equal,
 		},
+		{
+			sql: "drop table delete_me",
+			exp: tableResponse{
+				headers: []columnInfo{},
+				rows:    []row{},
+			},
+			eq: equal,
+		},
+		{
+			sql: "show tables",
+			exp: tableResponse{
+				headers: []columnInfo{
+					{"Table", "string"},
+				},
+				rows: []row{
+					{[]columnResponse{"grouper"}},
+					{[]columnResponse{"joiner"}},
+				},
+			},
+			eq: equal,
+		},
 	}
 
 	for i, test := range tests {
@@ -881,6 +903,9 @@ func setUpTestQuerySQLUnary(ctx context.Context, t *testing.T) (gh *server.GRPCH
 		}
 	}
 
+	// delete_me
+	m.MustCreateIndex(t, "delete_me", pilosa.IndexOptions{TrackExistence: true})
+
 	return gh, func() {
 		if err := m.API.DeleteIndex(ctx, joiner.Name()); err != nil {
 			panic(err)

server/handler_test.go

@@ -43,7 +43,7 @@ import (
 func TestHandler_PostSchemaCluster(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	h := cmd.Handler.(*http.Handler).Handler
 
 	t.Run("PostSchema", func(t *testing.T) {
@@ -56,8 +56,8 @@ func TestHandler_PostSchemaCluster(t *testing.T) {
 			}
 			t.Fatalf("unexpected code: %v, bod: %s", w.Code, bod)
 		}
-		for i := 0; i < len(cluster); i++ {
-			cmd = cluster[i]
+		for i := 0; i < cluster.Len(); i++ {
+			cmd = cluster.GetNode(i)
 			idx, err := cmd.API.Index(context.Background(), "blah")
 			if err != nil {
 				t.Fatalf("getting index: %v", err)
@@ -82,7 +82,7 @@ func TestHandler_PostSchemaCluster(t *testing.T) {
 func TestHandler_Endpoints(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	h := cmd.Handler.(*http.Handler).Handler
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -1049,7 +1049,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})})
 		defer clus.Close()
 		w = httptest.NewRecorder()
-		h := clus[0].Handler.(*http.Handler).Handler
+		h := clus.GetNode(0).Handler.(*http.Handler).Handler
 		h.ServeHTTP(w, req)
 		result = w.Result()
 
@@ -1297,59 +1297,59 @@ func TestHandler_Endpoints(t *testing.T) {
 }
 
 func TestCluster_TranslateStore(t *testing.T) {
-	cluster := make(test.Cluster, 1)
-	cluster[0] = test.NewCommandNode(true,
+	cluster := test.MustNewCluster(t, 1)
+	cluster.Nodes[0] = test.NewCommandNode(t, true,
 		server.OptCommandServerOptions(
 			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
 			pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})),
 		),
 	)
-	cluster[0].Config.Gossip.Port = "0"
-	err := cluster[0].Start()
+	cluster.GetNode(0).Config.Gossip.Port = "0"
+	err := cluster.GetNode(0).Start()
 	if err != nil {
 		t.Fatalf("starting cluster 0: %v", err)
 	}
-	defer cluster[0].Close()
+	defer cluster.GetNode(0).Close()
 
-	test.Do(t, "POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
 }
 
 func TestClusterTranslator(t *testing.T) {
-	cluster := make(test.Cluster, 2)
-	cluster[0] = test.NewCommandNode(true,
+	cluster := test.MustNewCluster(t, 2)
+	cluster.Nodes[0] = test.NewCommandNode(t, true,
 		server.OptCommandServerOptions(
 			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
 		),
 	)
-	cluster[0].Config.Gossip.Port = "0"
-	err := cluster[0].Start()
+	cluster.GetNode(0).Config.Gossip.Port = "0"
+	err := cluster.GetNode(0).Start()
 	if err != nil {
 		t.Fatalf("starting cluster 0: %v", err)
 	}
-	defer cluster[0].Close()
-	cluster[1] = test.NewCommandNode(false,
+	defer cluster.GetNode(0).Close()
+	cluster.Nodes[1] = test.NewCommandNode(t, false,
 		server.OptCommandServerOptions(
 			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
 			pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})),
 		),
 	)
-	cluster[1].Config.Gossip.Port = "0"
-	cluster[1].Config.Gossip.Seeds = []string{cluster[0].GossipAddress()}
-	err = cluster[1].Start()
+	cluster.GetNode(1).Config.Gossip.Port = "0"
+	cluster.GetNode(1).Config.Gossip.Seeds = []string{cluster.GetNode(0).GossipAddress()}
+	err = cluster.GetNode(1).Start()
 	if err != nil {
 		t.Fatalf("starting cluster 1: %v", err)
 	}
-	defer cluster[1].Close()
+	defer cluster.GetNode(1).Close()
 
-	test.Do(t, "POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
-	test.Do(t, "POST", cluster[0].URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}")
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}")
 
-	test.Do(t, "POST", cluster[0].URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")")
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")")
 
 	var result0, result1 string
 	if err := test.RetryUntil(2*time.Second, func() error {
-		result0 = test.Do(t, "POST", cluster[0].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
-		result1 = test.Do(t, "POST", cluster[1].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
+		result0 = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
+		result1 = test.Do(t, "POST", cluster.GetNode(1).URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
 		if result0 != result1 {
 			return fmt.Errorf("`%s` != `%s`", result0, result1)
 		}

server/server.go

@@ -49,6 +49,7 @@ import (
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pilosa/pilosa/v2/statsd"
 	"github.com/pilosa/pilosa/v2/syswrap"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -196,6 +197,7 @@ func (m *Command) Start() (err error) {
 		}
 	}
 
+	_ = testhook.Opened(pilosa.NewAuditor(), m, nil)
 	close(m.Started)
 	return nil
 }
@@ -537,6 +539,7 @@ func (m *Command) GossipTransport() *gossip.Transport {
 func (m *Command) Close() error {
 	defer close(m.done)
 	eg := errgroup.Group{}
+	m.grpcServer.Stop()
 	eg.Go(m.Handler.Close)
 	eg.Go(m.Server.Close)
 	eg.Go(m.API.Close)
@@ -552,6 +555,7 @@ func (m *Command) Close() error {
 	}
 
 	err := eg.Wait()
+	_ = testhook.Closed(pilosa.NewAuditor(), m, nil)
 	return errors.Wrap(err, "closing everything")
 }
 

server/server_test.go

@@ -355,7 +355,7 @@ func TestConcurrentFieldCreation(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
 
-	api0 := cluster[0].API
+	api0 := cluster.GetNode(0).API
 	if _, err := api0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
@@ -379,10 +379,10 @@ func TestTransactionsAPI(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
 
-	api0 := cluster[0].API
-	api1 := cluster[1].API
+	api0 := cluster.GetNode(0).API
+	api1 := cluster.GetNode(1).API
 	ctx := context.Background()
-	//api2 := cluster[2].API
+	//api2 := cluster.GetNode(2).API
 
 	// can fetch empty transactions
 	if trnsMap, err := api0.Transactions(ctx); err != nil {
@@ -508,7 +508,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
 	defer cluster.Close()
 
 	// Create the schema.
-	client0 := cluster[0].Client()
+	client0 := cluster.GetNode(0).Client()
 	if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
 		t.Fatal("create index:", err)
 	}
@@ -523,12 +523,12 @@ func TestMain_RecalculateHashes(t *testing.T) {
 			data = append(data, fmt.Sprintf(`Set(%d, f=%d)`, columnID, rowID))
 		}
 	}
-	if _, err := cluster[0].Query(t, "i", "", strings.Join(data, "")); err != nil {
+	if _, err := cluster.GetNode(0).Query(t, "i", "", strings.Join(data, "")); err != nil {
 		t.Fatal("setting columns:", err)
 	}
 
 	// Calculate caches on the first node
-	err := cluster[0].RecalculateCaches(t)
+	err := cluster.GetNode(0).RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating caches: %v", err)
 	}
@@ -536,7 +536,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
 	target := `{"results":[[{"id":7,"key":"","count":99},{"id":1,"key":"","count":99},{"id":9,"key":"","count":99},{"id":5,"key":"","count":99},{"id":4,"key":"","count":99},{"id":8,"key":"","count":99},{"id":2,"key":"","count":99},{"id":6,"key":"","count":99},{"id":3,"key":"","count":99}]]}`
 
 	// Run a TopN query on all nodes. The result should be the same as the target.
-	for _, m := range cluster {
+	for _, m := range cluster.Nodes {
 		res, err := m.Query(t, "i", "", `TopN(f)`)
 		if err != nil {
 			t.Fatal(err)
@@ -635,27 +635,27 @@ func TestClusteringNodesReplica1(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	if err := cluster[2].Command.Close(); err != nil {
+	if err := cluster.GetNode(2).Command.Close(); err != nil {
 		t.Fatalf("closing third node: %v", err)
 	}
 
 	// confirm that cluster stops accepting queries after one node closes
-	if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
+	if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
 		t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
 	}
 
 	// Create new main with the same config.
-	config := cluster[2].Command.Config
+	config := cluster.GetNode(2).Command.Config
 	config.Translation.MapSize = 100000
 
 	// this isn't necessary, but makes the test run way faster
-	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
+	config.Gossip.Port = strconv.Itoa(int(cluster.GetNode(2).Command.GossipTransport().URI.Port))
 
-	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
-	cluster[2].Command.Config = config
+	cluster.GetNode(2).Command = server.NewCommand(cluster.GetNode(2).Stdin, cluster.GetNode(2).Stdout, cluster.GetNode(2).Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
+	cluster.GetNode(2).Command.Config = config
 
 	// Run new program.
-	if err := cluster[2].Start(); err != nil {
+	if err := cluster.GetNode(2).Start(); err != nil {
 		t.Fatalf("restarting node 2: %v", err)
 	}
 
@@ -667,7 +667,7 @@ func TestClusteringNodesReplica1(t *testing.T) {
 
 func TestClusteringNodesReplica2(t *testing.T) {
 	cluster := test.MustNewCluster(t, 3)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -681,7 +681,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	if err := cluster[2].Command.Close(); err != nil {
+	if err := cluster.GetNode(2).Command.Close(); err != nil {
 		t.Fatalf("closing third node: %v", err)
 	}
 
@@ -691,12 +691,12 @@ func TestClusteringNodesReplica2(t *testing.T) {
 	}
 
 	// confirm that cluster keeps accepting queries if replication > 1
-	if _, err := cluster[0].API.CreateIndex(context.Background(), "anewindex", pilosa.IndexOptions{}); err != nil {
+	if _, err := cluster.GetNode(0).API.CreateIndex(context.Background(), "anewindex", pilosa.IndexOptions{}); err != nil {
 		t.Fatalf("got unexpected error creating index: %v", err)
 	}
 
 	// confirm that cluster stops accepting queries if 2 nodes fail and replication == 2
-	if err := cluster[1].Command.Close(); err != nil {
+	if err := cluster.GetNode(1).Command.Close(); err != nil {
 		t.Fatalf("closing 2nd node: %v", err)
 	}
 
@@ -705,23 +705,23 @@ func TestClusteringNodesReplica2(t *testing.T) {
 		t.Fatalf("after closing second server: %v", err)
 	}
 
-	if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
+	if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
 		t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
 	}
 
 	// Create new main with the same config.
-	config := cluster[2].Command.Config
+	config := cluster.GetNode(2).Command.Config
 	config.Translation.MapSize = 100000
-	// config.Bind = cluster[2].API.Node().URI.HostPort()
+	// config.Bind = cluster.GetNode(2).API.Node().URI.HostPort()
 
 	// this isn't necessary, but makes the test run way faster
-	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
+	config.Gossip.Port = strconv.Itoa(int(cluster.GetNode(2).Command.GossipTransport().URI.Port))
 
-	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
-	cluster[2].Command.Config = config
+	cluster.GetNode(2).Command = server.NewCommand(cluster.GetNode(2).Stdin, cluster.GetNode(2).Stdout, cluster.GetNode(2).Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
+	cluster.GetNode(2).Command.Config = config
 
 	// Run new program.
-	if err := cluster[2].Start(); err != nil {
+	if err := cluster.GetNode(2).Start(); err != nil {
 		t.Fatalf("restarting node 2: %v", err)
 	}
 
@@ -731,18 +731,18 @@ func TestClusteringNodesReplica2(t *testing.T) {
 	}
 
 	// Create new main with the same config.
-	config = cluster[1].Command.Config
-	// config.Bind = cluster[1].API.Node().URI.HostPort()
+	config = cluster.GetNode(1).Command.Config
+	// config.Bind = cluster.GetNode(1).API.Node().URI.HostPort()
 	config.Translation.MapSize = 100000
 
 	// this isn't necessary, but makes the test run way faster
-	config.Gossip.Port = strconv.Itoa(int(cluster[1].Command.GossipTransport().URI.Port))
+	config.Gossip.Port = strconv.Itoa(int(cluster.GetNode(1).Command.GossipTransport().URI.Port))
 
-	cluster[1].Command = server.NewCommand(cluster[1].Stdin, cluster[1].Stdout, cluster[1].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
-	cluster[1].Command.Config = config
+	cluster.GetNode(1).Command = server.NewCommand(cluster.GetNode(1).Stdin, cluster.GetNode(1).Stdout, cluster.GetNode(1).Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
+	cluster.GetNode(1).Command.Config = config
 
 	// Run new program.
-	if err := cluster[1].Start(); err != nil {
+	if err := cluster.GetNode(1).Start(); err != nil {
 		t.Fatalf("restarting node 1: %v", err)
 	}
 
@@ -754,7 +754,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
 
 func TestRemoveNodeAfterItDies(t *testing.T) {
 	cluster := test.MustNewCluster(t, 3)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -774,10 +774,9 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	// prevent double-closing cluster[2] from the deferred Close above
-	disabled, cluster := cluster[2], cluster[:2]
-
-	if err := disabled.Command.Close(); err != nil {
+	// prevent double-closing cluster.GetNode(2) from the deferred Close above
+	disabled := cluster.GetNode(2)
+	if err := cluster.CloseAndRemove(2); err != nil {
 		t.Fatalf("closing third node: %v", err)
 	}
 
@@ -786,7 +785,7 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	if _, err := cluster[0].API.RemoveNode(disabled.API.Node().ID); err != nil {
+	if _, err := cluster.GetNode(0).API.RemoveNode(disabled.API.Node().ID); err != nil {
 		t.Fatalf("removing failed node: %v", err)
 	}
 
@@ -795,7 +794,7 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
 		t.Fatalf("removing disabled node: %v", err)
 	}
 
-	hosts := cluster[0].API.Hosts(context.Background())
+	hosts := cluster.GetNode(0).API.Hosts(context.Background())
 	if len(hosts) != 2 {
 		t.Fatalf("unexpected hosts: %v", hosts)
 	}
@@ -803,7 +802,7 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
 
 func TestRemoveConcurrentIndexCreation(t *testing.T) {
 	cluster := test.MustNewCluster(t, 3)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -818,11 +817,11 @@ func TestRemoveConcurrentIndexCreation(t *testing.T) {
 
 	errc := make(chan error)
 	go func() {
-		_, err := cluster[0].API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{})
+		_, err := cluster.GetNode(0).API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{})
 		errc <- err
 	}()
 
-	if _, err := cluster[0].API.RemoveNode(cluster[2].API.Node().ID); err != nil {
+	if _, err := cluster.GetNode(0).API.RemoveNode(cluster.GetNode(2).API.Node().ID); err != nil {
 		t.Fatalf("removing node: %v", err)
 	}
 
@@ -831,7 +830,7 @@ func TestRemoveConcurrentIndexCreation(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	hosts := cluster[0].API.Hosts(context.Background())
+	hosts := cluster.GetNode(0).API.Hosts(context.Background())
 	if len(hosts) != 2 {
 		t.Fatalf("unexpected hosts: %v", hosts)
 	}
@@ -948,9 +947,9 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) {
 func TestClusterQueriesAfterRestart(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
-	cmd1 := cluster[1]
+	cmd1 := cluster.GetNode(1)
 
-	for _, com := range cluster {
+	for _, com := range cluster.Nodes {
 		nodes := com.API.Hosts(context.Background())
 		for _, n := range nodes {
 			if n.State != "READY" {
@@ -992,7 +991,7 @@ func TestClusterQueriesAfterRestart(t *testing.T) {
 	}
 
 	// confirm that cluster stops accepting queries after one node closes
-	if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
+	if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
 		t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
 	}
 
@@ -1034,9 +1033,9 @@ func TestClusterExhaustingConnections(t *testing.T) {
 	}
 	cluster := test.MustRunCluster(t, 5)
 	defer cluster.Close()
-	cmd1 := cluster[1]
+	cmd1 := cluster.GetNode(1)
 
-	for _, com := range cluster {
+	for _, com := range cluster.Nodes {
 		nodes := com.API.Hosts(context.Background())
 		for _, n := range nodes {
 			if n.State != "READY" {
@@ -1053,7 +1052,7 @@ func TestClusterExhaustingConnections(t *testing.T) {
 		i := i
 		eg.Go(func() error {
 			for j := i; j < 10000; j += 20 {
-				_, err := cluster[i%5].API.Query(context.Background(), &pilosa.QueryRequest{
+				_, err := cluster.GetNode(i%5).API.Query(context.Background(), &pilosa.QueryRequest{
 					Index: "testidx",
 					Query: fmt.Sprintf("Set(%d, testfield=0)", j*pilosa.ShardWidth),
 				})
@@ -1120,9 +1119,9 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) {
 	}
 	cluster := test.MustRunCluster(t, 5)
 	defer cluster.Close()
-	cmd1 := cluster[1]
+	cmd1 := cluster.GetNode(1)
 
-	for _, com := range cluster {
+	for _, com := range cluster.Nodes {
 		nodes := com.API.Hosts(context.Background())
 		for _, n := range nodes {
 			if n.State != "READY" {
@@ -1151,7 +1150,7 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) {
 				if (j-i)%1000 == 0 {
 					fmt.Printf("%d is %.2f%% done.\n", i, float64(j-i)*100/100000)
 				}
-				err := cluster[i%5].API.ImportRoaring(context.Background(), "testidx", "testfield", j, false, &pilosa.ImportRoaringRequest{
+				err := cluster.GetNode(int(i%5)).API.ImportRoaring(context.Background(), "testidx", "testfield", j, false, &pilosa.ImportRoaringRequest{
 					Views: map[string][]byte{
 						"": data,
 					},
@@ -1172,12 +1171,12 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) {
 func TestClusterMinMaxSumDecimal(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	cmd.MustCreateIndex(t, "testdec", pilosa.IndexOptions{Keys: true, TrackExistence: true})
 	cmd.MustCreateField(t, "testdec", "adec", pilosa.OptFieldTypeDecimal(2))
 
-	test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", `
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", `
 Set("a", adec=42.2)
 Set("b", adec=11.12)
 Set("c", adec=13.41)
@@ -1188,21 +1187,21 @@ Set("g", adec=15.52)
 Set("h", adec=100.22)
 `)
 
-	result := test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", "Sum(field=adec)")
+	result := test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Sum(field=adec)")
 	if !strings.Contains(result.Body, `"decimalValue":305.59`) {
 		t.Fatalf("expected decimal sum of 305.59, but got: '%s'", result.Body)
 	} else if !strings.Contains(result.Body, `"count":8`) {
 		t.Fatalf("expected count 8, but got: '%s'", result.Body)
 	}
 
-	result = test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", "Max(field=adec)")
+	result = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Max(field=adec)")
 	if !strings.Contains(result.Body, `"decimalValue":100.22`) {
 		t.Fatalf("expected decimal max of 100.22, but got: '%s'", result.Body)
 	} else if !strings.Contains(result.Body, `"count":1`) {
 		t.Fatalf("expected count 1, but got: '%s'", result.Body)
 	}
 
-	result = test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", "Min(field=adec)")
+	result = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Min(field=adec)")
 	if !strings.Contains(result.Body, `"decimalValue":11.12`) {
 		t.Fatalf("expected decimal min of 11.12, but got: '%s'", result.Body)
 	} else if !strings.Contains(result.Body, `"count":1`) {

server/sql.go

@@ -38,17 +38,14 @@ func execSQL(ctx context.Context, api *pilosa.API, logger logger.Logger, querySt
 	case sql.SQLTypeSelect:
 		handler := sql.NewSelectHandler(api)
 		results, err = handler.Handle(ctx, query)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to start SQL query")
-		}
 	case sql.SQLTypeShow:
 		handler := sql.NewShowHandler(api)
 		results, err = handler.Handle(ctx, query)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to start SQL query")
-		}
+	case sql.SQLTypeEmpty:
+		handler := sql.NewDDLHandler(api)
+		results, err = handler.Handle(ctx, query)
 	default:
 		return nil, status.Errorf(codes.Unimplemented, "query type not supported")
 	}
-	return results, nil
+	return results, errors.Wrap(err, "failed to start SQL query")
 }

server_internal_test.go

@@ -15,10 +15,11 @@
 package pilosa
 
 import (
-	"io/ioutil"
 	"runtime"
 	"testing"
 	"time"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure the file handle count is working
@@ -40,7 +41,7 @@ func TestCountOpenFiles(t *testing.T) {
 
 func TestMonitorAntiEntropyZero(t *testing.T) {
 
-	td, err := ioutil.TempDir(*TempDir, "")
+	td, err := testhook.TempDirInDir(t, *TempDir, "")
 	if err != nil {
 		t.Fatalf("getting temp dir: %v", err)
 	}
@@ -49,6 +50,7 @@ func TestMonitorAntiEntropyZero(t *testing.T) {
 	if err != nil {
 		t.Fatalf("making new server: %v", err)
 	}
+	defer s.Close()
 
 	ch := make(chan struct{})
 	go func() {

snapshotqueue.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -90,7 +91,7 @@ var defaultSnapshotQueue = &queuelessSnapshotQueue{}
 // w worker threads.
 func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue {
 	ctx, cancel := context.WithCancel(context.Background())
-	sq := prioritySnapshotQueue{
+	sq := &prioritySnapshotQueue{
 		normal:     make(chan snapshotRequest, n),
 		urgent:     make(chan snapshotRequest),
 		background: make(chan snapshotRequest),
@@ -102,8 +103,9 @@ func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue {
 	if sq.logger == nil {
 		sq.logger = logger.NewStandardLogger(os.Stderr)
 	}
+	_ = testhook.Opened(NewAuditor(), sq, nil)
 	sq.spawnWorkers(w)
-	return &sq
+	return sq
 }
 
 type snapshotRequest struct {
@@ -136,6 +138,7 @@ type prioritySnapshotQueue struct {
 		enqueued uint32
 		skipped  uint32
 	}
+	stopped bool
 }
 
 func (sq *prioritySnapshotQueue) spawnWorkers(w int) {
@@ -205,6 +208,10 @@ func (sq *prioritySnapshotQueue) process(req snapshotRequest) {
 func (sq *prioritySnapshotQueue) Stop() {
 	sq.mu.Lock()
 	defer sq.mu.Unlock()
+	if sq.stopped {
+		return
+	}
+	sq.stopped = true
 	sq.cancel()
 	// scanners need to be done before we close the other channels.
 	sq.scanWG.Wait()
@@ -214,6 +221,7 @@ func (sq *prioritySnapshotQueue) Stop() {
 	sq.urgent = nil
 	close(sq.background)
 	sq.background = nil
+	_ = testhook.Closed(NewAuditor(), sq, nil)
 	enqueued := atomic.LoadUint32(&sq.stats.enqueued)
 	skipped := atomic.LoadUint32(&sq.stats.skipped)
 	if skipped > 0 || enqueued > 1 {

sql/ddl.go

@@ -0,0 +1,65 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sql
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pilosa/pilosa/v2"
+	pproto "github.com/pilosa/pilosa/v2/proto"
+	"github.com/pkg/errors"
+	"vitess.io/vitess/go/vt/sqlparser"
+)
+
+// DDLHandler executes  CREATE, ALTER, DROP, RENAME, TRUNCATE or ANALYZE statement.
+type DDLHandler struct {
+	api *pilosa.API
+}
+
+// NewDDLHandler constructor
+func NewDDLHandler(api *pilosa.API) *DDLHandler {
+	return &DDLHandler{
+		api: api,
+	}
+}
+
+// Handle executes mapped SQL
+func (h *DDLHandler) Handle(ctx context.Context, mapped *MappedSQL) (pproto.StreamClient, error) {
+	stmt, ok := mapped.Statement.(*sqlparser.DDL)
+	if !ok {
+		return nil, fmt.Errorf("statement is not type DDL: %T", mapped.Statement)
+	}
+
+	switch stmt.Action {
+	case sqlparser.DropStr:
+		return h.execDropTable(ctx, stmt)
+
+	default:
+		return nil, errors.Errorf("unsupported DDL action: %s", stmt.Action)
+	}
+}
+
+func (h *DDLHandler) execDropTable(ctx context.Context, stmt *sqlparser.DDL) (pproto.StreamClient, error) {
+	if n := len(stmt.FromTables); n != 1 {
+		return nil, fmt.Errorf("statement can only contain a single drop table, but got: %d", n)
+	}
+
+	indexName := stmt.FromTables[0].ToViewName().Name.String()
+	if err := h.api.DeleteIndex(ctx, indexName); err != nil {
+		return nil, errors.Wrapf(err, "deleting index %s", indexName)
+	}
+	return pproto.EmptyStream{}, nil
+}

stats/stats_test.go

@@ -32,7 +32,7 @@ import (
 // TestMultiStatClient_Expvar run the multistat client with exp var
 // since the EXPVAR data is stored in a global we should run these in one test function
 func TestMultiStatClient_Expvar(t *testing.T) {
-	hldr := test.MustOpenHolder()
+	hldr := test.MustOpenHolder(t)
 	defer hldr.Close()
 
 	c := stats.NewExpvarStatsClient()
@@ -93,7 +93,7 @@ func TestMultiStatClient_Expvar(t *testing.T) {
 func TestStatsCount_TopN(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-	hldr := test.Holder{Holder: c[0].Server.Holder()}
+	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
 
 	hldr.SetBit("d", "f", 0, 0)
 	hldr.SetBit("d", "f", 0, 1)
@@ -115,7 +115,7 @@ func TestStatsCount_TopN(t *testing.T) {
 			called = true
 		},
 	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil {
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil {
 		t.Fatal(err)
 	}
 	if !called {
@@ -126,7 +126,7 @@ func TestStatsCount_TopN(t *testing.T) {
 func TestStatsCount_Bitmap(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-	hldr := test.Holder{Holder: c[0].Server.Holder()}
+	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
 
 	hldr.SetBit("d", "f", 0, 0)
 	hldr.SetBit("d", "f", 0, 1)
@@ -144,7 +144,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
 			called = true
 		},
 	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil {
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil {
 		t.Fatal(err)
 	}
 	if !called {
@@ -155,7 +155,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
 func TestStatsCount_SetRowAttrsBulk(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-	hldr := test.Holder{Holder: c[0].Server.Holder()}
+	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
 
 	hldr.SetBit("d", "f", 10, 0)
 	hldr.SetBit("d", "f", 10, 1)
@@ -178,7 +178,7 @@ func TestStatsCount_SetRowAttrsBulk(t *testing.T) {
 			called = true
 		},
 	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil {
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil {
 		t.Fatal(err)
 	}
 	if !called {
@@ -189,7 +189,7 @@ func TestStatsCount_SetRowAttrsBulk(t *testing.T) {
 func TestStatsCount_SetColumnAttrs(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-	hldr := test.Holder{Holder: c[0].Server.Holder()}
+	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
 
 	hldr.SetBit("d", "f", 10, 0)
 	hldr.SetBit("d", "f", 10, 1)
@@ -212,7 +212,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
 			called = true
 		},
 	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetColumnAttrs(10, foo="bar")`}); err != nil {
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetColumnAttrs(10, foo="bar")`}); err != nil {
 		t.Fatal(err)
 	}
 	if !called {
@@ -223,7 +223,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
 func TestStatsCount_APICalls(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	h := cmd.Handler.(*http.Handler).Handler
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}

test/cluster.go

@@ -35,20 +35,34 @@ type ModHasher struct{}
 func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
 
 // Cluster represents a Pilosa cluster (multiple Command instances)
-type Cluster []*Command
+type Cluster struct {
+	Nodes []*Command
+}
 
 // Query executes an API.Query through one of the cluster's node's API. It fails
 // the test if there is an error.
-func (c Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse {
+func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse {
 	t.Helper()
-	if len(c) == 0 {
+	if len(c.Nodes) == 0 {
 		t.Fatal("must have at least one node in cluster to query")
 	}
 
-	return c[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query})
+	return c.Nodes[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query})
 }
 
-func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) {
+func (c *Cluster) GetNode(n int) *Command {
+	return c.Nodes[n]
+}
+
+func (c *Cluster) GetHolder(n int) *Holder {
+	return &Holder{Holder: c.Nodes[n].Server.Holder()}
+}
+
+func (c *Cluster) Len() int {
+	return len(c.Nodes)
+}
+
+func (c *Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) {
 	t.Helper()
 	byShard := make(map[uint64][][2]uint64)
 	for _, rowcol := range rowcols {
@@ -63,7 +77,7 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint
 			rowIDs[i] = bit[0]
 			colIDs[i] = bit[1]
 		}
-		nodes, err := c[0].API.ShardNodes(context.Background(), index, shard)
+		nodes, err := c.Nodes[0].API.ShardNodes(context.Background(), index, shard)
 		if err != nil {
 			t.Fatalf("getting shard nodes: %v", err)
 		}
@@ -72,7 +86,7 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint
 		// suggesting that elsewhere we would support importing to a
 		// single node, regardless of where the data ends up.
 		for _, node := range nodes {
-			for _, com := range c {
+			for _, com := range c.Nodes {
 				if com.API.Node().ID != node.ID {
 					continue
 				}
@@ -92,13 +106,13 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint
 }
 
 // CreateField creates the index (if necessary) and field specified.
-func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field {
+func (c *Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field {
 	t.Helper()
-	idx, err := c[0].API.CreateIndex(context.Background(), index, iopts)
+	idx, err := c.Nodes[0].API.CreateIndex(context.Background(), index, iopts)
 	if err != nil && !strings.Contains(err.Error(), "index already exists") {
 		t.Fatalf("creating index: %v", err)
 	} else if err != nil { // index exists
-		idx, err = c[0].API.Index(context.Background(), index)
+		idx, err = c.Nodes[0].API.Index(context.Background(), index)
 		if err != nil {
 			t.Fatalf("getting index: %v", err)
 		}
@@ -107,7 +121,7 @@ func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptio
 		t.Logf("existing index options:\n%v\ndon't match given opts:\n%v\n in pilosa/test.Cluster.CreateField", idx.Options(), iopts)
 	}
 
-	f, err := c[0].API.CreateField(context.Background(), index, field, fopts...)
+	f, err := c.Nodes[0].API.CreateField(context.Background(), index, field, fopts...)
 	// we'll assume the field doesn't exist because checking if the options
 	// match seems painful.
 	if err != nil {
@@ -117,9 +131,9 @@ func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptio
 }
 
 // Start runs a Cluster
-func (c Cluster) Start() error {
-	var gossipSeeds = make([]string, len(c))
-	for i, cc := range c {
+func (c *Cluster) Start() error {
+	var gossipSeeds = make([]string, len(c.Nodes))
+	for i, cc := range c.Nodes {
 		cc.Config.Gossip.Port = "0"
 		cc.Config.Gossip.Seeds = gossipSeeds[:i]
 		if err := cc.Start(); err != nil {
@@ -131,8 +145,8 @@ func (c Cluster) Start() error {
 }
 
 // Stop stops a Cluster
-func (c Cluster) Close() error {
-	for i, cc := range c {
+func (c *Cluster) Close() error {
+	for i, cc := range c.Nodes {
 		if err := cc.Close(); err != nil {
 			return errors.Wrapf(err, "stopping server %d", i)
 		}
@@ -140,19 +154,30 @@ func (c Cluster) Close() error {
 	return nil
 }
 
+func (c *Cluster) CloseAndRemove(n int) error {
+	if n < 0 || n >= len(c.Nodes) {
+		return fmt.Errorf("close/remove from cluster: index %d out of range (len %d)", n, len(c.Nodes))
+	}
+	err := c.Nodes[n].Close()
+	copy(c.Nodes[n:], c.Nodes[n+1:])
+	c.Nodes = c.Nodes[:len(c.Nodes)-1]
+	return err
+}
+
 // AwaitState waits for the cluster coordinator (assumed to be the first
 // node) to reach a specified state.
-func (c Cluster) AwaitCoordinatorState(expectedState string, timeout time.Duration) error {
-	if len(c) < 1 {
+func (c *Cluster) AwaitCoordinatorState(expectedState string, timeout time.Duration) error {
+	if len(c.Nodes) < 1 {
 		return errors.New("can't await coordinator state on an empty cluster")
 	}
-	return c[:1].AwaitState(expectedState, timeout)
+	onlyCoordinator := &Cluster{Nodes: c.Nodes[:1]}
+	return onlyCoordinator.AwaitState(expectedState, timeout)
 }
 
 // ExceptionalState returns an error if any node in the cluster is not
 // in the expected state.
-func (c Cluster) ExceptionalState(expectedState string) error {
-	for _, node := range c {
+func (c *Cluster) ExceptionalState(expectedState string) error {
+	for _, node := range c.Nodes {
 		state := node.API.State()
 		if state != expectedState {
 			return fmt.Errorf("node %q: state %s", node.ID(), state)
@@ -162,8 +187,8 @@ func (c Cluster) ExceptionalState(expectedState string) error {
 }
 
 // AwaitState waits for the whole cluster to reach a specified state.
-func (c Cluster) AwaitState(expectedState string, timeout time.Duration) (err error) {
-	if len(c) < 1 {
+func (c *Cluster) AwaitState(expectedState string, timeout time.Duration) (err error) {
+	if len(c.Nodes) < 1 {
 		return errors.New("can't await state of an empty cluster")
 	}
 	startTime := time.Now()
@@ -184,7 +209,7 @@ func (c Cluster) AwaitState(expectedState string, timeout time.Duration) (err er
 // slice of command options, those options are used with every node.
 // If it is empty, default options are used. Otherwise, it must contain size
 // slices of command options, which are used with corresponding nodes.
-func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
+func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cluster {
 	tb.Helper()
 	c, err := newCluster(tb, size, opts...)
 	if err != nil {
@@ -206,7 +231,7 @@ func CheckClusterState(m *Command, state string, n int) bool {
 }
 
 // newCluster creates a new cluster
-func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluster, error) {
+func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Cluster, error) {
 	if size == 0 {
 		return nil, errors.New("cluster must contain at least one node")
 	}
@@ -214,26 +239,26 @@ func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluste
 		return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
 	}
 
-	cluster := make(Cluster, size)
+	cluster := &Cluster{Nodes: make([]*Command, size)}
 	name := tb.Name()
 	for i := 0; i < size; i++ {
 		var commandOpts []server.CommandOption
 		if len(opts) > 0 {
 			commandOpts = opts[i%len(opts)]
 		}
-		m := NewCommandNode(i == 0, commandOpts...)
+		m := NewCommandNode(tb, i == 0, commandOpts...)
 		err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name+"_"+strconv.Itoa(i)), 0600)
 		if err != nil {
 			return nil, errors.Wrap(err, "writing node id")
 		}
-		cluster[i] = m
+		cluster.Nodes[i] = m
 	}
 
 	return cluster, nil
 }
 
 // runCluster creates and starts a new cluster
-func runCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluster, error) {
+func runCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Cluster, error) {
 	cluster, err := newCluster(tb, size, opts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "new cluster")
@@ -247,7 +272,7 @@ func runCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluste
 
 // MustRunCluster creates and starts a new cluster. The opts parameter
 // is slightly magical; see MustNewCluster.
-func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
+func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cluster {
 	// We want tests to default to using the in-memory translate store, so we
 	// prepend opts with that functional option. If a different translate store
 	// has been specified, it will override this one.

test/field.go

@@ -15,11 +15,11 @@
 package test
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Field represents a test wrapper for pilosa.Field.
@@ -28,8 +28,8 @@ type Field struct {
 }
 
 // newField returns a new instance of Field d/0.
-func newField(opts pilosa.FieldOption) *Field {
-	path, err := ioutil.TempDir("", "pilosa-field-")
+func newField(tb testing.TB, opts pilosa.FieldOption) *Field {
+	path, err := testhook.TempDir(tb, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
@@ -41,8 +41,8 @@ func newField(opts pilosa.FieldOption) *Field {
 }
 
 // mustOpenField returns a new, opened field at a temporary path. Panic on error.
-func mustOpenField(opts pilosa.FieldOption) *Field {
-	f := newField(opts)
+func mustOpenField(tb testing.TB, opts pilosa.FieldOption) *Field {
+	f := newField(tb, opts)
 	if err := f.Open(); err != nil {
 		panic(err)
 	}
@@ -76,7 +76,7 @@ func (f *Field) reopen() error {
 
 // Ensure field can set its cache
 func TestField_SetCacheSize(t *testing.T) {
-	f := mustOpenField(pilosa.OptFieldTypeDefault())
+	f := mustOpenField(t, pilosa.OptFieldTypeDefault())
 	defer f.close()
 	cacheSize := uint32(100)
 

test/holder.go

@@ -15,14 +15,14 @@
 package test
 
 import (
-	"io/ioutil"
 	"math"
-	"os"
+	"testing"
 	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/boltdb"
 	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 var panicOn = pilosa.PanicOn
@@ -33,8 +33,8 @@ type Holder struct {
 }
 
 // NewHolder returns a new instance of Holder with a temporary path.
-func NewHolder() *Holder {
-	path, err := ioutil.TempDir("", "pilosa-")
+func NewHolder(tb testing.TB) *Holder {
+	path, err := testhook.TempDir(tb, "pilosa-holder-")
 	if err != nil {
 		panic(err)
 	}
@@ -47,17 +47,16 @@ func NewHolder() *Holder {
 }
 
 // MustOpenHolder creates and opens a holder at a temporary path. Panic on error.
-func MustOpenHolder() *Holder {
-	h := NewHolder()
+func MustOpenHolder(tb testing.TB) *Holder {
+	h := NewHolder(tb)
 	if err := h.Open(); err != nil {
 		panic(err)
 	}
 	return h
 }
 
-// Close closes the holder and removes all underlying data.
+// Close closes the holder. The data should be removed by the
 func (h *Holder) Close() error {
-	defer os.RemoveAll(h.Path)
 	return h.Holder.Close()
 }
 

test/index.go

@@ -15,10 +15,10 @@
 package test
 
 import (
-	"io/ioutil"
-	"os"
+	"testing"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Index represents a test wrapper for pilosa.Index.
@@ -27,12 +27,15 @@ type Index struct {
 }
 
 // newIndex returns a new instance of Index.
-func newIndex() *Index {
-	path, err := ioutil.TempDir("", "pilosa-index-")
+func newIndex(tb testing.TB) *Index {
+	path, err := testhook.TempDir(tb, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}
 	h := pilosa.NewHolder(pilosa.DefaultPartitionN)
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	h.Path = path
 	index, err := h.CreateIndex("i", pilosa.IndexOptions{})
 	if err != nil {
@@ -42,17 +45,13 @@ func newIndex() *Index {
 }
 
 // MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
-func MustOpenIndex() *Index {
-	index := newIndex()
-	if err := index.Open(false); err != nil {
-		panic(err)
-	}
+func MustOpenIndex(tb testing.TB) *Index {
+	index := newIndex(tb)
 	return index
 }
 
 // Close closes the index and removes the underlying data.
 func (i *Index) Close() error {
-	defer os.RemoveAll(i.Path())
 	return i.Index.Close()
 }
 
@@ -70,10 +69,6 @@ func (i *Index) Reopen() error {
 	if err != nil {
 		return err
 	}
-
-	if err := i.Open(false); err != nil {
-		return err
-	}
 	return nil
 }
 

test/pilosa.go

@@ -30,6 +30,7 @@ import (
 	"github.com/pilosa/pilosa/v2/encoding/proto"
 	"github.com/pilosa/pilosa/v2/http"
 	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////
@@ -48,8 +49,8 @@ func OptAllowedOrigins(origins []string) server.CommandOption {
 }
 
 // newCommand returns a new instance of Main with a temporary data directory and random port.
-func newCommand(opts ...server.CommandOption) *Command {
-	path, err := ioutil.TempDir("", "pilosa-")
+func newCommand(tb testing.TB, opts ...server.CommandOption) *Command {
+	path, err := testhook.TempDir(tb, "pilosa-command-")
 	if err != nil {
 		panic(err)
 	}
@@ -85,12 +86,12 @@ func newCommand(opts ...server.CommandOption) *Command {
 }
 
 // NewCommandNode returns a new instance of Command with clustering enabled.
-func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
+func NewCommandNode(tb testing.TB, isCoordinator bool, opts ...server.CommandOption) *Command {
 	// We want tests to default to using the in-memory translate store, so we
 	// prepend opts with that functional option. If a different translate store
 	// has been specified, it will override this one.
 	opts = prependTestServerOpts(opts)
-	m := newCommand(opts...)
+	m := newCommand(tb, opts...)
 	m.Config.Cluster.Disabled = false
 	m.Config.Cluster.Coordinator = isCoordinator
 	return m
@@ -99,7 +100,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
 // RunCommand returns a new, running Main. Panic on error.
 func RunCommand(t *testing.T) *Command {
 	t.Helper()
-	m := newCommand(server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
+	m := newCommand(t, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
 	m.Config.Metric.Diagnostics = false // Disable diagnostics.
 	m.Config.Gossip.Port = "0"
 	if err := m.Start(); err != nil {
@@ -307,8 +308,15 @@ func Do(t *testing.T, method, urlStr string, body string) *httpResponse {
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 
-	resp, err := gohttp.DefaultClient.Do(req)
+	// set a timeout instead of allowing gohttp.Defaultclient to
+	// potentially hang forever.
+	hc := &gohttp.Client{
+		Timeout: time.Second * 10,
+	}
+	resp, err := hc.Do(req)
+
 	if err != nil {
+		fmt.Printf(" hc.Do() err = '%v'\n", err)
 		t.Fatal(err)
 	}
 	defer resp.Body.Close()

test/pilosa_test.go

@@ -30,15 +30,15 @@ func TestNewCluster(t *testing.T) {
 	cluster := test.MustRunCluster(t, numNodes)
 	defer cluster.Close()
 
-	coordinator := getCoordinator(cluster[0])
+	coordinator := getCoordinator(cluster.Nodes[0])
 	for i := 1; i < numNodes; i++ {
-		if coordi := getCoordinator(cluster[i]); coordi != coordinator {
+		if coordi := getCoordinator(cluster.Nodes[i]); coordi != coordinator {
 			t.Fatalf("node %d does not have the same coordinator as node 0. '%v' and '%v' respectively", i, coordi, coordinator)
 		}
 	}
 	req, err := http.NewRequest(
 		"GET",
-		cluster[0].URL()+"/status",
+		cluster.Nodes[0].URL()+"/status",
 		strings.NewReader(""),
 	)
 	if err != nil {

testhook/auditor.go

@@ -0,0 +1,168 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testhook
+
+import (
+	"fmt"
+	"reflect"
+	"sync"
+)
+
+// Auditor represents a thing which knows how to audit events. For instance,
+// it can check on when things are accessed, or when they were opened, or
+// whether opened objects are later closed.
+type Auditor interface {
+
+	// Registry yields a registry for objects of this type.
+	// multiple calls with the same object type yield the same registry.
+	Registry(interface{}) (Registry, error)
+	// Check performs any error-checking that can be done during
+	// usage.
+	Check() (error, []error)
+	// FinalCheck performs any error-checking that makes sense only
+	// after all operations are supposed to be complete, such as
+	// verifying that opened objects have been closed.
+	FinalCheck() (error, []error)
+}
+
+// Created(a, o, kv) is shorthand for a.Registry(o).Created(o, kv) plus
+// the error checking inside that.
+func Created(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Created(o, kv)
+}
+
+// Opened(a, o, kv) is shorthand for a.Registry(o).Opened(o, kv) plus
+// the error checking inside that.
+func Opened(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Opened(o, kv)
+}
+
+// Closed(a, o, kv) is shorthand for a.Registry(o).Closed(o, kv) plus
+// the error checking inside that.
+func Closed(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Closed(o, kv)
+}
+
+// Destroyed(a, o, kv) is shorthand for a.Registry(o).Destroyed(o, kv) plus
+// the error checking inside that.
+func Destroyed(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Destroyed(o, kv)
+}
+
+// Seen(a, o, kv) is shorthand for a.Registry(o).Seen(o, kv) plus
+// the error checking inside that.
+func Seen(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Seen(o, kv)
+}
+
+// NopAuditor doesn't do anything.
+type NopAuditor struct{}
+
+func (*NopAuditor) Registry(interface{}) (Registry, error) {
+	return NewNopRegistry(), nil
+}
+
+func (*NopAuditor) Check() (error, []error) {
+	return nil, nil
+}
+
+func (*NopAuditor) FinalCheck() (error, []error) {
+	return nil, nil
+}
+
+func NewNopAuditor() *NopAuditor {
+	return &NopAuditor{}
+}
+
+// VerifyCloseAuditor provides registries which it will check for things
+// being closed.
+type VerifyCloseAuditor struct {
+	registries map[reflect.Type]Registry
+	hooks      RegistryHooks
+	regMu      sync.Mutex
+}
+
+func (v *VerifyCloseAuditor) Registry(o interface{}) (Registry, error) {
+	t := reflect.TypeOf(o)
+	v.regMu.Lock()
+	defer v.regMu.Unlock()
+	if exists, ok := v.registries[t]; ok {
+		return exists, nil
+	}
+	reg := NewSimpleRegistry(v.hooks[t])
+	v.registries[t] = reg
+	return reg, nil
+}
+
+func (*VerifyCloseAuditor) Check() (error, []error) {
+	return nil, nil
+}
+
+func (v *VerifyCloseAuditor) FinalCheck() (error, []error) {
+	v.regMu.Lock()
+	defer v.regMu.Unlock()
+	var errs []error
+	for t, reg := range v.registries {
+		typeName := t.String()
+		live, err := reg.Live()
+		if err != nil {
+			errs = append(errs, fmt.Errorf("registry[%s]: retrieving live list: %v",
+				typeName, err))
+			continue
+		}
+		if len(live) > 0 {
+			for addr, entry := range live {
+				if entry.Error != nil {
+					errs = append(errs, fmt.Errorf("%v: item created at %v, stack %s",
+						entry.Error, entry.Stamp, entry.Stack))
+				} else {
+					errs = append(errs, fmt.Errorf("live item found at %p, created at %v, stack %s",
+						addr, entry.Stamp, entry.Stack))
+				}
+				if entry.Data["stack"] != nil {
+					errs = append(errs, fmt.Errorf("stashed stack: %s", entry.Data["stack"]))
+				}
+			}
+		}
+	}
+	if len(errs) > 0 {
+		return fmt.Errorf("final check: %d error(s)", len(errs)), errs
+	}
+	return nil, nil
+}
+
+func NewVerifyCloseAuditor(hooks RegistryHooks) *VerifyCloseAuditor {
+	return &VerifyCloseAuditor{registries: map[reflect.Type]Registry{}, hooks: hooks}
+}

testhook/auditor_test.go

@@ -0,0 +1,331 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testhook_test
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+func TestAuditor_CatchError(t *testing.T) {
+	auditor := testhook.NewVerifyCloseAuditor(nil)
+	var x, y int
+	reg, err := auditor.Registry(&x)
+	if err != nil {
+		t.Fatalf("requesting registry: %v", err)
+	}
+	err = reg.Created(&x, nil)
+	if err != nil {
+		t.Fatalf("creating x: %v", err)
+	}
+	err = reg.Opened(&x, nil)
+	if err != nil {
+		t.Fatalf("opening x: %v", err)
+	}
+	err = reg.Seen(&y, nil)
+	if err == nil {
+		t.Fatalf("seeing unopened y: expected error, didn't get one")
+	}
+	err = reg.Seen(&x, nil)
+	if err != nil {
+		t.Fatalf("seeing opened x: unexpected error %v", err)
+	}
+	err = reg.Created(&y, nil)
+	if err != nil {
+		t.Fatalf("creating y: %v", err)
+	}
+	err = reg.Opened(&y, nil)
+	if err != nil {
+		t.Fatalf("opening y: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err != nil {
+		t.Fatalf("closing x: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err == nil {
+		t.Fatalf("double-closing x: expected error, didn't get one")
+	}
+	err = reg.Destroyed(&x, nil)
+	if err != nil {
+		t.Fatalf("destroying x: got unexpected error %v", err)
+	}
+	err, errs := auditor.FinalCheck()
+	if err == nil {
+		t.Fatalf("unclosed y not detected")
+	}
+	_ = errs
+}
+
+type ignoreLiveness struct {
+	skippable *int
+}
+
+func (ign *ignoreLiveness) Live(o interface{}, _ *testhook.RegistryEntry) error {
+	if ptr, ok := o.(*int); ok {
+		if ptr == ign.skippable {
+			return nil
+		}
+	}
+	return errors.New("unexpected live object")
+}
+
+var _ testhook.RegistryHookLive = &ignoreLiveness{}
+
+func TestAuditor_DiscardError(t *testing.T) {
+	var x, y int
+	iptr := reflect.TypeOf(&x)
+	auditor := testhook.NewVerifyCloseAuditor(testhook.RegistryHooks{iptr: &ignoreLiveness{skippable: &y}})
+	reg, err := auditor.Registry(&x)
+	if err != nil {
+		t.Fatalf("requesting registry: %v", err)
+	}
+	err = reg.Created(&x, nil)
+	if err != nil {
+		t.Fatalf("creating x: %v", err)
+	}
+	err = reg.Opened(&x, nil)
+	if err != nil {
+		t.Fatalf("opening x: %v", err)
+	}
+	err = reg.Seen(&y, nil)
+	if err == nil {
+		t.Fatalf("seeing unopened y: expected error, didn't get one")
+	}
+	err = reg.Seen(&x, nil)
+	if err != nil {
+		t.Fatalf("seeing opened x: unexpected error %v", err)
+	}
+	err = reg.Created(&y, nil)
+	if err != nil {
+		t.Fatalf("creating y: %v", err)
+	}
+	err = reg.Opened(&y, nil)
+	if err != nil {
+		t.Fatalf("opening y: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err != nil {
+		t.Fatalf("closing x: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err == nil {
+		t.Fatalf("double-closing x: expected error, didn't get one")
+	}
+	err = reg.Destroyed(&x, nil)
+	if err != nil {
+		t.Fatalf("destroying x: got unexpected error %v", err)
+	}
+	err, errs := auditor.FinalCheck()
+	if err != nil {
+		t.Fatalf("expected to skip y, instead got err %v, error list %v", err, errs)
+	}
+}
+
+func TestAuditor_KeepError(t *testing.T) {
+	var x, y, z int
+	iptr := reflect.TypeOf(&x)
+	auditor := testhook.NewVerifyCloseAuditor(testhook.RegistryHooks{iptr: &ignoreLiveness{skippable: &z}})
+	reg, err := auditor.Registry(&x)
+	if err != nil {
+		t.Fatalf("requesting registry: %v", err)
+	}
+	err = reg.Created(&x, nil)
+	if err != nil {
+		t.Fatalf("creating x: %v", err)
+	}
+	err = reg.Opened(&x, nil)
+	if err != nil {
+		t.Fatalf("opening x: %v", err)
+	}
+	err = reg.Seen(&y, nil)
+	if err == nil {
+		t.Fatalf("seeing unopened y: expected error, didn't get one")
+	}
+	err = reg.Seen(&x, nil)
+	if err != nil {
+		t.Fatalf("seeing opened x: unexpected error %v", err)
+	}
+	err = reg.Created(&y, nil)
+	if err != nil {
+		t.Fatalf("creating y: %v", err)
+	}
+	err = reg.Opened(&y, nil)
+	if err != nil {
+		t.Fatalf("opening y: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err != nil {
+		t.Fatalf("closing x: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err == nil {
+		t.Fatalf("double-closing x: expected error, didn't get one")
+	}
+	err = reg.Destroyed(&x, nil)
+	if err != nil {
+		t.Fatalf("destroying x: %v", err)
+	}
+	err, errs := auditor.FinalCheck()
+	if err == nil {
+		t.Fatalf("undestroyed y not detected")
+	}
+	_ = errs
+}
+
+type failHook struct {
+	calls int
+}
+
+func (f *failHook) Opened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+func (f *failHook) WasOpened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+// Implement WasSeen but not Seen, so we can verify that the only-one
+// case works in both directions.
+func (f *failHook) WasSeen(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+func (f *failHook) Closed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+func (f *failHook) WasClosed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+func (f *failHook) Live(i interface{}, ent *testhook.RegistryEntry) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+type successHook struct {
+	calls int
+}
+
+func (s *successHook) Opened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	s.calls++
+	return nil
+}
+
+func (s *successHook) WasOpened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	s.calls++
+	return nil
+}
+
+func (s *successHook) Closed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	s.calls++
+	return nil
+}
+
+func (s *successHook) Seen(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	s.calls++
+	return nil
+}
+
+// successHook allows an error to leak from WasClosed.
+func (s *successHook) WasClosed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	s.calls++
+	return err
+}
+
+func (s *successHook) Live(i interface{}, ent *testhook.RegistryEntry) error {
+	s.calls++
+	return errors.New("even successHook can fail sometimes")
+}
+
+func TestComposedHooks(t *testing.T) {
+	s := &successHook{}
+	f := &failHook{}
+	sExp, fExp := 0, 0
+	var err error
+	checkExp := func(when string) {
+		if s.calls != sExp {
+			t.Fatalf("after %s, expected %d calls to successHook, got %d", when, sExp, s.calls)
+		}
+		if f.calls != fExp {
+			t.Fatalf("after %s, expected %d calls to failHook, got %d", when, fExp, f.calls)
+		}
+	}
+	combined := testhook.Compose(s, f)
+	// Opened: we expect both opened calls to be hit, and the error from
+	// the second to come back.
+	err = combined.Opened(nil, nil, nil)
+	sExp++
+	fExp++
+	checkExp("opened")
+	if err == nil {
+		t.Fatalf("composed hook, opened: didn't error")
+	}
+	if err.Error() != "failHook always fails" {
+		t.Fatalf("composed hook, opened: expected failHook always fails, got %v", err)
+	}
+
+	// WasOpened: we expect the error from failHook to be overridden.
+	err = combined.WasOpened(nil, nil, nil, nil)
+	sExp++
+	fExp++
+	checkExp("wasOpened")
+	if err != nil {
+		t.Fatalf("composed hook, wasOpened: expected no error, got %v", err)
+	}
+
+	// Seen: nothing to call for failHook
+	err = combined.Seen(nil, nil, nil)
+	sExp++
+	checkExp("seen")
+	if err != nil {
+		t.Fatalf("composed hook, seen: expected no error, got %v", err)
+	}
+
+	// WasSeen: nothing to call for successHook
+	err = combined.WasSeen(nil, nil, nil, nil)
+	fExp++
+	checkExp("wasSeen")
+	if err == nil {
+		t.Fatalf("composed hook, wasSeen: expected error, didn't get it")
+	}
+
+	// WasClosed: expect both to get called, but successHook to leak the error up
+	err = combined.WasClosed(nil, nil, nil, nil)
+	sExp++
+	fExp++
+	checkExp("wasClosed")
+	if err == nil {
+		t.Fatalf("composed hook, wasClosed: expected error, didn't get it")
+	}
+
+	// Live: the failure from successHook (oops) should prevent failHook from
+	// being called.
+	err = combined.Live(nil, nil)
+	sExp++
+	checkExp("live")
+	if err == nil {
+		t.Fatalf("composed hook, live: expected error, didn't get it")
+	}
+}

testhook/cleanup1.13.go

@@ -0,0 +1,47 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !go1.14
+
+package testhook
+
+import (
+	"sync"
+	"testing"
+)
+
+var cleanupFuncs []func()
+var cleanupMu sync.Mutex
+
+func init() {
+	RegisterPostTestHook(runCleanupFuncs)
+}
+
+func runCleanupFuncs() error {
+	cleanupMu.Lock()
+	defer cleanupMu.Unlock()
+	for _, fn := range cleanupFuncs {
+		fn()
+	}
+	return nil
+}
+
+// Cleanup in 1.13 logs a message about skipping a cleanup function, but
+// allows things to build. Cleanup in 1.14 uses tb.Cleanup to register
+// a cleanup function to call when a test completes.
+func Cleanup(tb testing.TB, fn func()) {
+	cleanupMu.Lock()
+	defer cleanupMu.Unlock()
+	cleanupFuncs = append(cleanupFuncs, fn)
+}

testhook/cleanup1.14.go

@@ -0,0 +1,28 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.14
+
+package testhook
+
+import (
+	"testing"
+)
+
+// Cleanup in 1.13 logs a message about skipping a cleanup function, but
+// allows things to build. Cleanup in 1.14 uses tb.Cleanup to register
+// a cleanup function to call when a test completes.
+func Cleanup(tb testing.TB, fn func()) {
+	tb.Cleanup(fn)
+}

testhook/hook.go

@@ -0,0 +1,107 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testhook
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sync"
+	"testing"
+)
+
+// Callback denotes a function which can be run on a testing.T, or testing.B,
+// which performs additional functions typically before or after tests.
+type Callback func() error
+
+var preHooks []Callback
+var postHooks []Callback
+var mu sync.Mutex
+
+// RegisterPostTestHook registers a function to be called after tests
+// are run. It should return a nil error if it's okay, and a non-nil
+// error to cause a non-zero exit status.
+func RegisterPostTestHook(fn Callback) {
+	mu.Lock()
+	defer mu.Unlock()
+	postHooks = append(postHooks, fn)
+}
+
+// RegisterPreTestHook registers a function to be called after tests
+// are run. It should return a nil error if it's okay, and a non-nil
+// error to cause a non-zero exit status.
+func RegisterPreTestHook(fn Callback) {
+	mu.Lock()
+	defer mu.Unlock()
+	preHooks = append(preHooks, fn)
+}
+
+// RunTestsWithHooks is a suitable implementation for TestMain; you can
+// just invoke this from your TestMain, passing in m, and it runs the tests
+// and then runs any registered pre/post hooks. If the hooks themselves try
+// to register hooks, you will deadlock. Don't do that.
+func RunTestsWithHooks(m *testing.M) {
+	var ret int
+	mu.Lock()
+	for _, fn := range preHooks {
+		err := fn()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "pre-hook failure: %v\n", err)
+			ret = 1
+		}
+	}
+	mu.Unlock()
+	if ret != 0 {
+		fmt.Fprint(os.Stderr, "pre-hooks failed, aborting.\n")
+		os.Exit(ret)
+	}
+	ret = m.Run()
+	mu.Lock()
+	defer mu.Unlock()
+	for _, fn := range postHooks {
+		err := fn()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "post-hook failure: %v\n", err)
+			ret = 1
+		}
+	}
+	os.Exit(ret)
+}
+
+// TempDir creates a temp directory that will be automatically deleted when
+// this test completes, using go1.14's [TB].Cleanup() if available.
+func TempDir(tb testing.TB, pattern string) (path string, err error) {
+	path, err = ioutil.TempDir("", pattern)
+	if err == nil {
+		Cleanup(tb, func() {
+			os.RemoveAll(path)
+		})
+	}
+	return path, err
+}
+
+// TempDirInDir creates a temp directory that will be automatically deleted when
+// this test completes, using go1.14's [TB].Cleanup(), but with a specified
+// path instead of the default Go TMPDIR. Only some tests use this, which is
+// possibly an error...
+func TempDirInDir(tb testing.TB, dir string, pattern string) (path string, err error) {
+	path, err = ioutil.TempDir(dir, pattern)
+	if err == nil {
+		Cleanup(tb, func() {
+			os.RemoveAll(path)
+		})
+	}
+	return path, err
+}

testhook/registry.go

@@ -0,0 +1,524 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testhook
+
+import (
+	"fmt"
+	"reflect"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+// KV represents a key/value mapping. You don't need to use the type
+// for anything, it's just to make typing easier.
+type KV map[string]interface{}
+
+// Registry represents a set of known objects of a given common type.
+// A typical implementation might maintain a map of objects it's seen which
+// haven't been deleted. A minimal implementation just does nothing.
+//
+// A Registry may implement reference-count semantics, or may treat
+// reopening of a still-open object as an error.
+//
+// All objects provided to a registry should have the same type.
+type Registry interface {
+	Created(interface{}, KV) error                 // Object has been created, and must be destroyed later.
+	Opened(interface{}, KV) error                  // Object has been opened, and must be closed later.
+	Seen(interface{}, KV) error                    // Object has been interacted with, and should be between an open and a close.
+	Closed(interface{}, KV) error                  // Object has been closed, and must have been opened previously.
+	Destroyed(interface{}, KV) error               // Object has been destroyed
+	Live() (map[interface{}]*RegistryEntry, error) // Currently live objects
+}
+
+// RegistryHook is a generic interface, but any real implementation should
+// implement at least one of RegistryHookPreOpen, RegistryHookPostOpen,
+// etcetera. Pre-hooks are called before any error checks by the registry,
+// post-hooks are called with after those error checks, and are passed the
+// (possibly nil) error that would be returned at that point. If the post-hook
+// overrides the error, the registry continues with operations as though it
+// hadn't occurred, which may be a very bad idea.
+type RegistryHook interface{}
+
+type RegistryHookPreCreate interface {
+	Created(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostCreate interface {
+	WasCreated(interface{}, KV, *RegistryEntry, error) error
+}
+
+type RegistryHookPreOpen interface {
+	Opened(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostOpen interface {
+	WasOpened(interface{}, KV, *RegistryEntry, error) error
+}
+
+type RegistryHookPreSee interface {
+	Seen(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostSee interface {
+	WasSeen(interface{}, KV, *RegistryEntry, error) error
+}
+
+type RegistryHookPreClose interface {
+	Closed(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostClose interface {
+	WasClosed(interface{}, KV, *RegistryEntry, error) error
+}
+
+type RegistryHookPreDestroy interface {
+	Destroyed(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostDestroy interface {
+	WasDestroyed(interface{}, KV, *RegistryEntry, error) error
+}
+
+// If a registry's hooks implement RegistryHookLive, Live() should
+// return only those entries for which a non-nil error was returned, with the
+// error inserted in the RegistryEntry.
+type RegistryHookLive interface {
+	Live(interface{}, *RegistryEntry) error
+}
+
+// RegistryHooks represents a set of registry hook values to use for
+// registries, corresponding to different types.
+type RegistryHooks map[reflect.Type]RegistryHook
+
+// RegistryEntry represents the data we might have about an entry. Every
+// entry in it could be zero-valued in some implementations
+type RegistryEntry struct {
+	Error     error
+	Stack     []byte
+	Stamp     time.Time
+	Data      KV
+	OpenCount int
+}
+
+// NopRegistry doesn't do anything; it exists to fit the interface but not
+// consume resources.
+type NopRegistry struct{}
+
+var _ Registry = &NopRegistry{}
+
+func (*NopRegistry) Created(interface{}, KV) error                 { return nil }
+func (*NopRegistry) Opened(interface{}, KV) error                  { return nil }
+func (*NopRegistry) Seen(interface{}, KV) error                    { return nil }
+func (*NopRegistry) Closed(interface{}, KV) error                  { return nil }
+func (*NopRegistry) Destroyed(interface{}, KV) error               { return nil }
+func (*NopRegistry) Live() (map[interface{}]*RegistryEntry, error) { return nil, nil }
+
+func NewNopRegistry() *NopRegistry {
+	return &NopRegistry{}
+}
+
+// SimpleRegistry asserts that objects are created, then opened, then
+// possibly seen, then closed, then destroyed, and that they are not
+// opened more than once at a time, or destroyed while open. As a
+// convenience feature for users whose use cases might rely on this,
+// it will actually accept a new item being opened without being
+// previously created; to prevent this, use a PreOpen hook that checks
+// for a nil *RegistryEntry. The default Live check will report only
+// objects with an open count other than 0, but if you provide a Live
+// hook, you can return errors for objects still existing.
+type SimpleRegistry struct {
+	hooks   RegistryHook
+	entries map[interface{}]*RegistryEntry
+	mu      sync.Mutex
+}
+
+var _ Registry = &SimpleRegistry{}
+
+func NewSimpleRegistry(hooks RegistryHook) *SimpleRegistry {
+	return &SimpleRegistry{entries: map[interface{}]*RegistryEntry{}, hooks: hooks}
+}
+
+func (s *SimpleRegistry) Created(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreCreate); ok {
+		if err := hook.Created(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing != nil {
+		err = fmt.Errorf("object %T:%v previously registered at %v", o, o, existing.Stamp)
+	} else {
+		existing = &RegistryEntry{
+			Stack: debug.Stack(),
+			Data:  kv,
+			Stamp: time.Now(),
+		}
+	}
+	if hook, ok := s.hooks.(RegistryHookPostCreate); ok {
+		err = hook.WasCreated(o, kv, existing, err)
+	}
+	if err != nil {
+		return err
+	}
+	// if you overrode the error, or there wasn't one and you didn't
+	// create one, we now stash the entry.
+	s.entries[o] = existing
+	return nil
+}
+
+func (s *SimpleRegistry) Opened(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreOpen); ok {
+		if err := hook.Opened(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing == nil {
+		existing = &RegistryEntry{
+			Stack: debug.Stack(),
+			Data:  kv,
+			Stamp: time.Now(),
+		}
+		s.entries[o] = existing
+	}
+	existing.OpenCount++
+	if existing.OpenCount > 1 {
+		err = fmt.Errorf("object %T:%v opened %d times", o, o, existing.OpenCount)
+	}
+	if hook, ok := s.hooks.(RegistryHookPostOpen); ok {
+		err = hook.WasOpened(o, kv, existing, err)
+	}
+	if err != nil {
+		if existing != nil {
+			existing.OpenCount--
+		}
+		return err
+	}
+	return nil
+}
+
+func (s *SimpleRegistry) Seen(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreSee); ok {
+		if err = hook.Seen(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing == nil {
+		err = fmt.Errorf("object %T:%v seen but not previously registered", o, o)
+	}
+	if hook, ok := s.hooks.(RegistryHookPostSee); ok {
+		err = hook.WasSeen(o, kv, existing, err)
+	}
+	return err
+}
+
+func (s *SimpleRegistry) Closed(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreClose); ok {
+		if err = hook.Closed(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing == nil {
+		err = fmt.Errorf("object %T:%v closed but not previously registered", o, o)
+	} else {
+		existing.OpenCount--
+		if existing.OpenCount < 0 {
+			err = fmt.Errorf("object %T:%v closed more often than it was open: %d", o, o, existing.OpenCount)
+		}
+	}
+	if hook, ok := s.hooks.(RegistryHookPostClose); ok {
+		err = hook.WasClosed(o, kv, existing, err)
+	}
+	if err != nil {
+		// if a close "failed", we don't want to count it as being closed.
+		if existing != nil {
+			existing.OpenCount++
+		}
+		return err
+	}
+	return nil
+}
+
+func (s *SimpleRegistry) Destroyed(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreDestroy); ok {
+		if err = hook.Destroyed(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing == nil {
+		err = fmt.Errorf("object %T:%v destroyed but not previously registered", o, o)
+	} else if existing.OpenCount != 0 {
+		err = fmt.Errorf("object %T:%v destroyed while open count is %d", o, o, existing.OpenCount)
+	}
+	if hook, ok := s.hooks.(RegistryHookPostDestroy); ok {
+		err = hook.WasDestroyed(o, kv, existing, err)
+	}
+	if err != nil {
+		fmt.Printf("destroyed error: %v\n", err)
+		return err
+	}
+	// remove the entry from the list.
+	delete(s.entries, o)
+	return nil
+}
+
+func (s *SimpleRegistry) Live() (results map[interface{}]*RegistryEntry, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	results = make(map[interface{}]*RegistryEntry)
+	if hook, ok := s.hooks.(RegistryHookLive); ok {
+		for k, v := range s.entries {
+			v.Error = hook.Live(k, v)
+			if v.Error != nil {
+				results[k] = v
+			}
+		}
+	} else {
+		for k, v := range s.entries {
+			if v.OpenCount != 0 {
+				v.Error = fmt.Errorf("open count %d", v.OpenCount)
+				results[k] = v
+			}
+		}
+	}
+	return results, nil
+}
+
+type PreHook func(interface{}, KV, *RegistryEntry) error
+type PostHook func(interface{}, KV, *RegistryEntry, error) error
+type LiveHook func(interface{}, *RegistryEntry) error
+
+// PluggableRegistry is a registry which takes dynamically-generated functions,
+// and calls them if they're not nil.
+type PluggableRegistry struct {
+	implCreated      PreHook
+	implWasCreated   PostHook
+	implOpened       PreHook
+	implWasOpened    PostHook
+	implSeen         PreHook
+	implWasSeen      PostHook
+	implClosed       PreHook
+	implWasClosed    PostHook
+	implDestroyed    PreHook
+	implWasDestroyed PostHook
+	implLive         LiveHook
+}
+
+func (p *PluggableRegistry) Created(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implCreated != nil {
+		return p.implCreated(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasCreated(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasCreated != nil {
+		return p.implWasCreated(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Opened(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implOpened != nil {
+		return p.implOpened(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasOpened(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasOpened != nil {
+		return p.implWasOpened(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Seen(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implSeen != nil {
+		return p.implSeen(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasSeen(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasSeen != nil {
+		return p.implWasSeen(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Closed(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implClosed != nil {
+		return p.implClosed(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasClosed(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasClosed != nil {
+		return p.implWasClosed(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Destroyed(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implDestroyed != nil {
+		return p.implDestroyed(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasDestroyed(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasDestroyed != nil {
+		return p.implWasDestroyed(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Live(i interface{}, ent *RegistryEntry) error {
+	if p.implLive != nil {
+		return p.implLive(i, ent)
+	}
+	return nil
+}
+
+func composePreHooks(fns ...PreHook) PreHook {
+	if len(fns) == 0 {
+		return nil
+	}
+	if len(fns) == 1 {
+		return fns[0]
+	}
+	return func(i interface{}, kv KV, ent *RegistryEntry) error {
+		for _, fn := range fns {
+			err := fn(i, kv, ent)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func composePostHooks(fns ...PostHook) PostHook {
+	if len(fns) == 0 {
+		return nil
+	}
+	if len(fns) == 1 {
+		return fns[0]
+	}
+	return func(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+		// We run the functions in reverse order, so the last
+		// hook added has the option of overriding a lower hook's
+		// opinion.
+		for i := range fns {
+			fn := fns[len(fns)-1-i]
+			err = fn(i, kv, ent, err)
+		}
+		return err
+	}
+}
+
+func composeLiveHooks(fns ...LiveHook) LiveHook {
+	if len(fns) == 0 {
+		return nil
+	}
+	if len(fns) == 1 {
+		return fns[0]
+	}
+	return func(i interface{}, ent *RegistryEntry) error {
+		for _, fn := range fns {
+			err := fn(i, ent)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+// Compose takes a list of RegistryHook objects, and combines their hook
+// functions into a unified registry. For Pre hooks and Live hooks, the
+// functions are called in the order they were provided to this function,
+// and the first one to return an error causes the remainder to be
+// skipped; for Post hooks, they are called in the opposite order, and
+// the whole list continues to be called, with each function getting passed
+// the error (or non-error) value returned by the previous one.
+func Compose(hooks ...RegistryHook) *PluggableRegistry {
+	var created, opened, seen, closed, destroyed []PreHook
+	var wasCreated, wasOpened, wasSeen, wasClosed, wasDestroyed []PostHook
+	var live []LiveHook
+	for _, h := range hooks {
+		if hook, ok := h.(RegistryHookPreCreate); ok {
+			created = append(created, hook.Created)
+		}
+		if hook, ok := h.(RegistryHookPostCreate); ok {
+			wasCreated = append(wasCreated, hook.WasCreated)
+		}
+		if hook, ok := h.(RegistryHookPreOpen); ok {
+			opened = append(opened, hook.Opened)
+		}
+		if hook, ok := h.(RegistryHookPostOpen); ok {
+			wasOpened = append(wasOpened, hook.WasOpened)
+		}
+		if hook, ok := h.(RegistryHookPreSee); ok {
+			seen = append(seen, hook.Seen)
+		}
+		if hook, ok := h.(RegistryHookPostSee); ok {
+			wasSeen = append(wasSeen, hook.WasSeen)
+		}
+		if hook, ok := h.(RegistryHookPreClose); ok {
+			closed = append(closed, hook.Closed)
+		}
+		if hook, ok := h.(RegistryHookPostClose); ok {
+			wasClosed = append(wasClosed, hook.WasClosed)
+		}
+		if hook, ok := h.(RegistryHookPreDestroy); ok {
+			destroyed = append(destroyed, hook.Destroyed)
+		}
+		if hook, ok := h.(RegistryHookPostDestroy); ok {
+			wasDestroyed = append(wasDestroyed, hook.WasDestroyed)
+		}
+		if hook, ok := h.(RegistryHookLive); ok {
+			live = append(live, hook.Live)
+		}
+	}
+	return &PluggableRegistry{
+		implCreated:      composePreHooks(created...),
+		implWasCreated:   composePostHooks(wasCreated...),
+		implOpened:       composePreHooks(opened...),
+		implWasOpened:    composePostHooks(wasOpened...),
+		implSeen:         composePreHooks(seen...),
+		implWasSeen:      composePostHooks(wasSeen...),
+		implClosed:       composePreHooks(closed...),
+		implWasClosed:    composePostHooks(wasClosed...),
+		implDestroyed:    composePreHooks(destroyed...),
+		implWasDestroyed: composePostHooks(wasDestroyed...),
+		implLive:         composeLiveHooks(live...),
+	}
+}

translator_test.go

@@ -227,11 +227,12 @@ func TestTranslation_Reset(t *testing.T) {
 					pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 				)},
 		)
+		defer c.Close()
 
-		node0 := c[0]
-		node1 := c[1]
-		node2 := c[2]
-		node3 := c[3]
+		node0 := c.GetNode(0)
+		node1 := c.GetNode(1)
+		node2 := c.GetNode(2)
+		node3 := c.GetNode(3)
 
 		ctx := context.Background()
 		idx := "i"
@@ -321,9 +322,10 @@ func TestTranslation_Replication(t *testing.T) {
 					pilosa.OptServerReplicaN(2),
 				)},
 		)
+		defer c.Close()
 
-		node0 := c[0]
-		node1 := c[1]
+		node0 := c.GetNode(0)
+		node1 := c.GetNode(1)
 
 		ctx := context.Background()
 		idx := "i"
@@ -362,7 +364,7 @@ func TestTranslation_Replication(t *testing.T) {
 		node0.QueryExpect(t, idx, "", `Row(f=1)`, exp)
 
 		// Kill one node.
-		if err := node1.Command.Close(); err != nil {
+		if err := c.CloseAndRemove(1); err != nil {
 			t.Fatal(err)
 		}
 
@@ -396,8 +398,8 @@ func TestTranslation_Coordinator(t *testing.T) {
 		)
 		defer c.Close()
 
-		node0 := c[0]
-		node1 := c[1]
+		node0 := c.GetNode(0)
+		node1 := c.GetNode(1)
 
 		ctx := context.Background()
 		idx := "i"

tx_test.go

@@ -58,6 +58,7 @@ func queryBalances(m0api *pilosa.API, acctOwnerID uint64, fldAcct0, fldAcct1, in
 	acct1bal = mustQueryAcct(m0api, acctOwnerID, fldAcct1, index)
 	return
 }
+
 func skipForRoaring(t *testing.T) {
 	src := os.Getenv("PILOSA_TXSRC")
 	// once txfactory.go DefaultTxsrc != RoaringTxn, this
@@ -67,7 +68,7 @@ func skipForRoaring(t *testing.T) {
 	}
 }
 
-func TestAPI_ImportAIR(t *testing.T) {
+func TestAPI_ImportAtomicRecord(t *testing.T) {
 	skipForRoaring(t)
 	c := test.MustRunCluster(t, 1,
 		[]server.CommandOption{
@@ -79,7 +80,7 @@ func TestAPI_ImportAIR(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
+	m0 := c.GetNode(0)
 	m0api := m0.API
 
 	ctx := context.Background()

utils_internal_test.go

@@ -19,15 +19,17 @@ import (
 	"io/ioutil"
 	"path/filepath"
 	"sync"
+	"testing"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
 // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
-func NewTestCluster(n int) *cluster {
-	path, err := ioutil.TempDir("", "pilosa-cluster-")
+func NewTestCluster(tb testing.TB, n int) *cluster {
+	path, err := testhook.TempDir(tb, "pilosa-cluster-")
 	if err != nil {
 		panic(err)
 	}
@@ -88,6 +90,7 @@ type ClusterCluster struct {
 	mu         sync.RWMutex
 	resizing   bool
 	resizeDone chan struct{}
+	tb         testing.TB
 }
 
 type commonClusterSettings struct {
@@ -226,7 +229,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error)
 	t.common.Nodes = append(t.common.Nodes, node)
 
 	// create node-specific temp directory
-	path, err := ioutil.TempDir(*TempDir, fmt.Sprintf("pilosa-cluster-node-%d-", i))
+	path, err := testhook.TempDirInDir(t.tb, *TempDir, fmt.Sprintf("pilosa-cluster-node-%d-", i))
 	if err != nil {
 		return nil, err
 	}
@@ -262,10 +265,11 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error)
 }
 
 // NewClusterCluster returns a new instance of test.Cluster.
-func NewClusterCluster(n int) *ClusterCluster {
+func NewClusterCluster(tb testing.TB, n int) *ClusterCluster {
 
 	tc := &ClusterCluster{
 		common: &commonClusterSettings{},
+		tb:     tb,
 	}
 
 	// add clusters

view.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
@@ -158,6 +159,7 @@ func (v *view) open() error {
 		return err
 	}
 
+	_ = testhook.Opened(v.holder.Auditor, v, nil)
 	v.holder.Logger.Debugf("successfully opened index/field/view: %s/%s/%s", v.index, v.field, v.name)
 	return nil
 }
@@ -224,6 +226,9 @@ shardLoop:
 func (v *view) close() error {
 	v.mu.Lock()
 	defer v.mu.Unlock()
+	defer func() {
+		_ = testhook.Closed(v.holder.Auditor, v, nil)
+	}()
 
 	// Close all fragments.
 	eg, ctx := errgroup.WithContext(context.Background())
@@ -394,6 +399,7 @@ func (v *view) deleteFragment(shard uint64) error {
 	v.holder.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)
 
 	idx := f.holder.Index(v.index)
+	f.Close()
 	if err := idx.Txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil {
 		return errors.Wrap(err, "DeleteFragment")
 	}

view_internal_test.go

@@ -15,16 +15,16 @@
 package pilosa
 
 import (
-	"io/ioutil"
 	"testing"
 	"time"
 
+	"github.com/pilosa/pilosa/v2/testhook"
 	"golang.org/x/sync/errgroup"
 )
 
 // mustOpenView returns a new instance of View with a temporary path.
-func mustOpenView(index, field, name string) *view {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-view-")
+func mustOpenView(tb testing.TB, index, field, name string) *view {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-view-")
 	if err != nil {
 		panic(err)
 	}
@@ -38,7 +38,9 @@ func mustOpenView(index, field, name string) *view {
 	h.Path = path
 	// h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment
 	idx, err := h.createIndex(index, IndexOptions{})
-	_ = idx
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	panicOn(err)
 
 	v := newView(h, path, index, field, name, fo)
@@ -54,7 +56,7 @@ func mustOpenView(index, field, name string) *view {
 
 // Ensure view can open and retrieve a fragment.
 func TestView_DeleteFragment(t *testing.T) {
-	v := mustOpenView("i", "f", "v")
+	v := mustOpenView(t, "i", "f", "v")
 	defer v.close()
 
 	shard := uint64(9)
@@ -89,7 +91,7 @@ func TestView_DeleteFragment(t *testing.T) {
 // if the broadcast operation takes a bit of time.
 func TestView_CreateFragmentRace(t *testing.T) {
 	var creates errgroup.Group
-	v := mustOpenView("i", "f", "v")
+	v := mustOpenView(t, "i", "f", "v")
 	defer v.close()
 
 	// Use a broadcaster which intentionally fails.