testhook: leak auditing infrastructure

The testhook/ package provides an easy way to set up multiple
hooks to run before/after tests are run.

The audit hooks track open and closes of storage backends,
files, indexes, and holders, for example. A tempdir wrapper
creates temporary directories which are automatically cleaned up
when the test ends. Any kind of resource creation that
should be closed at test conclusion can be tracked. We
will complain at the end of the TestMain if resources are
leaking.

Leaks under go1.13:

We use a wrapper function which is a no-op for go 1.13, but actually
calls testing.TB.Cleanup in go1.14, so we can still build with 1.13 even though
tests will leak files all over the place there. Because of this,
don't run the testhook tests when using 1.13, as they'll always fail.

- the test/pilosa.go http client now times out after 10 seconds
to help diagnose hung server situations.

- Makefile targets added to get better progress reports.

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
+.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
@@ -43,6 +43,33 @@ test:
 test-race:
 	go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v
 
+testv: topt testvsub
+
+testv-race: topt-race testvsub-race
+
+# testvsub: run go test -v in sub-directories in "local mode" with incremental output,
+#            avoiding go -test ./... "package list mode" which doesn't give output
+#            until the test run finishes. Package list mode makes it hard to
+#            find which test is hung/deadlocked.
+#
+testvsub:
+	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i"; \
+           cd $$i; pwd; \
+           go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v || break; \
+           echo; echo "999 done testing subpkg $$i"; \
+           cd ..; \
+        done
+
+testvsub-race:
+	set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \
+           echo; echo "___ testing subpkg $$i -race"; \
+           cd $$i; pwd; \
+           go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race || break; \
+           echo; echo "999 done testing subpkg $$i -race"; \
+           cd ..; \
+        done
+
 bench:
 	go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
 

api.go

@@ -1114,9 +1114,6 @@ func (api *API) ImportAtomicRecord(ctx context.Context, req *AtomicRecord, opts
 	return tx.Commit()
 }
 
-// This is a hide your face ugly hack, forced upon
-// us by the horrible invention of function based options
-// by the usually brilliant Rob Pike. - JEA
 func addClearToImportOptions(opts []ImportOption) []ImportOption {
 	var opt ImportOptions
 	for _, o := range opts {

api_test.go

@@ -59,8 +59,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 	t.Run("ImportColumnAttrs", func(t *testing.T) {
 		ctx := context.Background()
 		indexName := "i"
@@ -184,8 +184,8 @@ func TestAPI_Import(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 
 	t.Run("RowIDColumnKey", func(t *testing.T) {
 		ctx := context.Background()
@@ -293,8 +293,8 @@ func TestAPI_ImportValue(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
-	m1 := c[1]
+	m0 := c.GetNode(0)
+	m1 := c.GetNode(1)
 
 	t.Run("ValColumnKey", func(t *testing.T) {
 		ctx := context.Background()
@@ -492,7 +492,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
 	//  3. verifiy the clear is done.
 	//  repeat for ImportValueRequest and ImportValues()
 
-	m0 := c[0]
+	m0 := c.GetNode(0)
 	m0api := m0.API
 
 	ctx := context.Background()

audit.go

@@ -0,0 +1,25 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+var NewAuditor func() testhook.Auditor = NewNopAuditor
+
+func NewNopAuditor() testhook.Auditor {
+	return testhook.NewNopAuditor()
+}

audit_internal_test.go

@@ -0,0 +1,52 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorViewHooks struct{}
+type auditorFragmentHooks struct{}
+
+// static type checks
+var _ testhook.RegistryHookLive = &auditorViewHooks{}
+var _ testhook.RegistryHookLive = &auditorFragmentHooks{}
+
+func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("view %s still open", o.(*view).name)
+	}
+	return nil
+}
+
+func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("fragment %s still open", o.(*fragment).path)
+	}
+	return nil
+}
+
+func GetInternalTestHooks() testhook.RegistryHooks {
+	return map[reflect.Type]testhook.RegistryHook{
+		reflect.TypeOf((*view)(nil)):     &auditorViewHooks{},
+		reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{},
+	}
+}

audit_test.go

@@ -0,0 +1,107 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+// AuditLeaksOn is a global switch to turn on resource
+// leak checking at the end of a test run.
+var AuditLeaksOn = true
+
+// for tests, we use a single shared auditor used by all of the holders.
+var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks)
+
+// These audit hooks are desireable during testing, but not in
+// production.
+type auditorIndexHooks struct{}
+type auditorFieldHooks struct{}
+type auditorHolderHooks struct{}
+
+// static type checking
+var _ testhook.RegistryHookLive = &auditorIndexHooks{}
+var _ testhook.RegistryHookLive = &auditorFieldHooks{}
+var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{}
+var _ testhook.RegistryHookLive = &auditorHolderHooks{}
+
+var testHooks = map[reflect.Type]testhook.RegistryHook{
+	reflect.TypeOf((*pilosa.Index)(nil)):  &auditorIndexHooks{},
+	reflect.TypeOf((*pilosa.Field)(nil)):  &auditorFieldHooks{},
+	reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{},
+}
+
+func init() {
+	if !AuditLeaksOn {
+		return
+	}
+	for k, v := range pilosa.GetInternalTestHooks() {
+		testHooks[k] = v
+	}
+	testhook.RegisterPreTestHook(func() error {
+		pilosa.NewAuditor = NewTestAuditor
+		return nil
+	})
+	testhook.RegisterPostTestHook(func() error {
+		err, errs := globalTestAuditor.FinalCheck()
+		if err != nil {
+			for i, e := range errs {
+				fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e)
+			}
+		}
+		return err
+	})
+}
+
+func NewTestAuditor() testhook.Auditor {
+	return globalTestAuditor
+}
+
+func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name())
+	}
+	return nil
+}
+
+func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name())
+	}
+	return nil
+}
+
+func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error {
+	path := o.(*pilosa.Holder).Path
+	if path == "" {
+		fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n",
+			ent.Stack)
+	} else {
+		os.RemoveAll(o.(*pilosa.Holder).Path)
+	}
+	return err
+}
+
+func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error {
+	if entry != nil && entry.OpenCount != 0 {
+		return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path)
+	}
+	return nil
+}

badger.go

@@ -32,6 +32,7 @@ import (
 	badger "github.com/dgraph-io/badger/v2"
 	badgeroptions "github.com/dgraph-io/badger/v2/options"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/txkey"
 	"github.com/pkg/errors"
 )
@@ -329,6 +330,7 @@ func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, e
 		halt:   halt,
 		hasher: NewBlake3Hasher(),
 	}
+	_ = testhook.Opened(NewAuditor(), w, nil)
 	r.unprotectedRegister(w)
 
 	w.startStack = stack()
@@ -504,6 +506,7 @@ func (w *BadgerDBWrapper) Close() (err error) {
 		close(w.halt)
 		w.closed = true
 	}
+	_ = testhook.Closed(NewAuditor(), w, nil)
 	return w.db.Close()
 }
 

badger_test.go

@@ -18,12 +18,8 @@
 // See https://github.com/dgraph-io/badger/issues/1384 for any progress.
 // What we see is that the value-log allocations immediately run out of
 // memory. So we turn off 386 with a build tag to keep the .circleci happy.
-//
-// gendebug_test will have a TestMain if build tag generationdebug is on,
-// so we avoid conflicting with that debug scenario.
 
 // +build !386
-// +build !generationdebug
 
 package pilosa
 
@@ -36,10 +32,16 @@ import (
 
 	"github.com/dgraph-io/badger/v2"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
+	"github.com/pkg/errors"
 )
 
 var _ = &roaring.Bitmap{}
 
+func init() {
+	testhook.RegisterPostTestHook(reportTestBadgersNeedingClose)
+}
+
 // helpers, each runs their own new txn, and commits if a change/delete
 // was made. The txn is rolled back if it is just viewing the data.
 
@@ -1639,26 +1641,20 @@ func BenchmarkBadger_Write(b *testing.B) {
 	*/
 }
 
-func reportTestBadgersNeedingClose() {
+func reportTestBadgersNeedingClose() error {
 	globalBadgerReg.mu.Lock()
 	defer globalBadgerReg.mu.Unlock()
 	n := len(globalBadgerReg.mp)
-	if n > 0 {
-		AlwaysPrintf("*** these badgers are still open (n=%v):", n)
-		i := 0
-		for w := range globalBadgerReg.mp {
-			AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack)
-			i++
-		}
+	if n == 0 {
+		return nil
 	}
-}
-
-var _ = reportTestBadgersNeedingClose // happy linter
-
-func TestMain(m *testing.M) {
-	ret := m.Run()
-	//reportTestBadgersNeedingClose()
-	os.Exit(ret)
+	AlwaysPrintf("*** these badgers are still open (n=%v):", n)
+	i := 0
+	for w := range globalBadgerReg.mp {
+		AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack)
+		i++
+	}
+	return errors.New("unclosed badgers, contact Animal Control")
 }
 
 /*

blake3_test.go

@@ -21,6 +21,8 @@ import (
 	"testing"
 
 	"encoding/hex"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestBlake3Hasher(t *testing.T) {
@@ -49,7 +51,7 @@ func TestCryptoRandInt64(t *testing.T) {
 }
 
 func TestHashOfDir(t *testing.T) {
-	dir, err := ioutil.TempDir(".", "TestHashOfDir-dir")
+	dir, err := testhook.TempDir(t, "TestHashOfDir-dir")
 	panicOn(err)
 	b := dir + sep + "A" + sep + "B"
 	c := dir + sep + "A" + sep + "C"
@@ -59,7 +61,6 @@ func TestHashOfDir(t *testing.T) {
 	panicOn(ioutil.WriteFile(b+sep+"b_content", bmessage, 0644))
 	cmessage := []byte("hello C\n")
 	panicOn(ioutil.WriteFile(c+sep+"c_content", cmessage, 0644))
-	defer os.RemoveAll(dir)
 	hsh := HashOfDir(dir)
 
 	c2message := []byte("hello C2\n")

cluster_internal_test.go

@@ -17,7 +17,6 @@ package pilosa
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -35,6 +34,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pilosa/pilosa/v2/logger"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -91,14 +91,17 @@ func TestFragCombos(t *testing.T) {
 }
 
 // newIndexWithTempPath returns a new instance of Index.
-func newIndexWithTempPath(name string) *Index {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
+func newIndexWithTempPath(tb testing.TB, name string) *Index {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}
 	h := NewHolder(DefaultPartitionN)
 	h.Path = path
 	index, err := h.CreateIndex(name, IndexOptions{})
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	if err != nil {
 		panic(err)
 	}
@@ -158,7 +161,7 @@ func TestFragSources(t *testing.T) {
 	c5.addNodeBasicSorted(node2)
 	c5.addNodeBasicSorted(node3)
 
-	idx := newIndexWithTempPath("i")
+	idx := newIndexWithTempPath(t, "i")
 	defer idx.Close()
 
 	// Obtain transaction.
@@ -398,7 +401,7 @@ func TestHasher(t *testing.T) {
 
 // Ensure ContainsShards can find the actual shard list for node and index.
 func TestCluster_ContainsShards(t *testing.T) {
-	c := NewTestCluster(5)
+	c := NewTestCluster(t, 5)
 	c.ReplicaN = 3
 	shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2])
 
@@ -544,7 +547,7 @@ func TestCluster_Coordinator(t *testing.T) {
 }
 
 func TestCluster_Topology(t *testing.T) {
-	c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"}
+	c1 := NewTestCluster(t, 1) // automatically creates Node{ID: "node0"}
 
 	uri0 := NewTestURIFromHostPort("host0", 0)
 	uri1 := NewTestURIFromHostPort("host1", 0)
@@ -592,7 +595,7 @@ func TestCluster_Topology(t *testing.T) {
 func TestCluster_ResizeStates(t *testing.T) {
 
 	t.Run("Single node, no data", func(t *testing.T) {
-		tc := NewClusterCluster(1)
+		tc := NewClusterCluster(t, 1)
 
 		// Open TestCluster.
 		if err := tc.Open(); err != nil {
@@ -622,7 +625,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Single node, in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -654,7 +657,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Single node, not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -683,7 +686,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, no data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -725,7 +728,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, in/not in topology", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -774,7 +777,7 @@ func TestCluster_ResizeStates(t *testing.T) {
 	})
 
 	t.Run("Multiple nodes, with data", func(t *testing.T) {
-		tc := NewClusterCluster(0)
+		tc := NewClusterCluster(t, 0)
 		if err := tc.addNode(); err != nil {
 			t.Fatalf("adding node: %v", err)
 		}
@@ -927,7 +930,7 @@ func TestAE(t *testing.T) {
 // Ensures that coordinator can be changed.
 func TestCluster_UpdateCoordinator(t *testing.T) {
 	t.Run("UpdateCoordinator", func(t *testing.T) {
-		c := NewTestCluster(2)
+		c := NewTestCluster(t, 2)
 
 		oldNode := c.nodes[0]
 		newNode := c.nodes[1]

ctl/export_test.go

@@ -46,7 +46,7 @@ func TestExportCommand_Validation(t *testing.T) {
 func TestExportCommand_Run(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)

ctl/import_test.go

@@ -73,7 +73,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -102,7 +102,7 @@ func TestImportCommand_Basic(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		cm.Index = "i"
@@ -135,7 +135,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -177,7 +177,7 @@ func TestImportCommand_RunValue(t *testing.T) {
 
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		cm.Host = cmd.API.Node().URI.HostPort()
 
 		resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@@ -219,7 +219,7 @@ func TestImportCommand_RunKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -271,8 +271,8 @@ func TestImportCommand_KeyReplication(t *testing.T) {
 
 	c := test.MustRunCluster(t, 2)
 	defer c.Close()
-	cmd0 := c[0]
-	cmd1 := c[1]
+	cmd0 := c.GetNode(0)
+	cmd1 := c.GetNode(1)
 
 	host0 := cmd0.API.Node().URI.HostPort()
 	host1 := cmd1.API.Node().URI.HostPort()
@@ -339,7 +339,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`)))
@@ -365,7 +365,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) {
 func TestImportCommand_InvalidFile(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -453,7 +453,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
 func TestImportCommand_BugOverwriteValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	buf := bytes.Buffer{}
 	stdin, stdout, stderr := GetIO(buf)
@@ -529,7 +529,7 @@ func TestImportCommand_RunBool(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	cm.Host = cmd.API.Node().URI.HostPort()
 
 	resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))

diagnostics_internal_test.go

@@ -29,6 +29,7 @@ import (
 func TestDiagnosticsClient(t *testing.T) {
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)
@@ -121,6 +122,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) {
 			t.Fatalf("couldn't encode version response: %v", err)
 		}
 	}))
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector("localhost:10101")
@@ -158,6 +160,7 @@ func compareJSON(a, b []byte) (bool, error) {
 func BenchmarkDiagnostics(b *testing.B) {
 	// Mock server.
 	server := httptest.NewServer(nil)
+	defer server.Close()
 
 	// Create a new client.
 	d := newDiagnosticsCollector(server.URL)

executor.go

@@ -29,6 +29,7 @@ import (
 	pb "github.com/pilosa/pilosa/v2/proto"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/shardwidth"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 )
@@ -100,6 +101,7 @@ func newExecutor(opts ...executorOption) *executor {
 	// the few tests we've done at scale with concurrent query
 	// workloads. Possible that it could be smaller.
 	e.work = make(chan job, e.workerPoolSize)
+	_ = testhook.Opened(NewAuditor(), e, nil)
 	for i := 0; i < e.workerPoolSize; i++ {
 		e.workersWG.Add(1)
 		go func() {
@@ -114,6 +116,7 @@ func (e *executor) Close() error {
 	e.workMu.Lock()
 	defer e.workMu.Unlock()
 	e.shutdown = true
+	_ = testhook.Closed(NewAuditor(), e, nil)
 	close(e.work)
 	e.workersWG.Wait()
 	return nil

executor_internal_test.go

@@ -18,24 +18,25 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"strconv"
 	"strings"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 func TestExecutor_TranslateGroupByCall(t *testing.T) {
 	holder := NewHolder(DefaultPartitionN)
+	defer holder.Close()
 
-	cluster := NewTestCluster(1)
+	cluster := NewTestCluster(t, 1)
 
 	e := &executor{
 		Holder:  holder,
 		Cluster: cluster,
 	}
-	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
+	e.Holder.Path, _ = testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
 	err := e.Holder.Open()
 	if err != nil {
 		t.Fatalf("opening holder: %v", err)
@@ -139,9 +140,9 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) {
 
 	e := &executor{
 		Holder:  holder,
-		Cluster: NewTestCluster(1),
+		Cluster: NewTestCluster(t, 1),
 	}
-	e.Holder.Path, _ = ioutil.TempDir(*TempDir, "")
+	e.Holder.Path, _ = testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
 	if err := e.Holder.Open(); err != nil {
 		t.Fatalf("opening holder: %v", err)
 	}

field.go

@@ -35,6 +35,7 @@ import (
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -607,9 +608,11 @@ func (f *Field) Open() error {
 		return err
 	}
 
+	_ = testhook.Opened(f.holder.Auditor, f, nil)
 	f.holder.Logger.Debugf("successfully opened field index/field: %s/%s", f.index, f.name)
 	return nil
 }
+
 func blockingWriteAvailableShards(fieldPath string, availableShardBytes []byte) {
 	path := filepath.Join(fieldPath, ".available.shards")
 	// Create a temporary file to save to.
@@ -972,12 +975,14 @@ func (f *Field) applyOptions(opt FieldOptions) error {
 func (f *Field) Close() error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	defer func() {
+		_ = testhook.Closed(f.holder.Auditor, f, nil)
+	}()
 	// Shutdown the available shards writer
 	if f.doneChan != nil {
-		f.doneChan <- struct{}{}
+		close(f.doneChan)
 		f.wg.Wait()
 		close(f.availableShardChan)
-		close(f.doneChan)
 		f.availableShardChan = nil
 		f.doneChan = nil
 	}

field_internal_test.go

@@ -16,7 +16,6 @@ package pilosa
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math"
 	"os"
 	"path/filepath"
@@ -28,6 +27,7 @@ import (
 
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure a bsiGroup can adjust to its baseValue.
@@ -197,11 +197,13 @@ func TestField_DeleteView(t *testing.T) {
 // TestField represents a test wrapper for Field.
 type TestField struct {
 	*Field
+	parent *Index
+	tb     testing.TB
 }
 
 // NewTestField returns a new instance of TestField d/0.
 func NewTestField(t *testing.T, opts FieldOption) *TestField {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-field-")
+	path, err := testhook.TempDirInDir(t, *TempDir, "pilosa-field-")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -211,20 +213,20 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
 	if err != nil {
 		panic(err)
 	}
-	field, err := NewField(h, path, "i", "f", opts)
+	field, err := idx.CreateField("f", opts)
 	if err != nil {
 		t.Fatal(err)
 	}
-	field.idx = idx
-	return &TestField{Field: field}
+	tf := &TestField{Field: field, parent: idx, tb: t}
+	testhook.Cleanup(t, func() {
+		h.Close()
+	})
+	return tf
 }
 
 // OpenField returns a new, opened field at a temporary path.
 func OpenField(t *testing.T, opts FieldOption) *TestField {
 	f := NewTestField(t, opts)
-	if err := f.Open(); err != nil {
-		t.Fatal(err)
-	}
 	return f
 }
 
@@ -239,27 +241,16 @@ func (f *TestField) Close() error {
 
 // Reopen closes the index and reopens it.
 func (f *TestField) Reopen() error {
-	var err error
-	if err := f.Field.Close(); err != nil {
+	name := f.Field.Name()
+	if err := f.parent.Close(); err != nil {
+		f.parent = nil
 		return err
 	}
-
-	path, index, name := f.Path(), f.Index(), f.Name()
-	h := NewHolder(DefaultPartitionN)
-	h.Path = path
-	idx, err := h.CreateIndex(index, IndexOptions{})
-	if err != nil {
-		return err
-	}
-	f.Field, err = NewField(h, path, index, name, OptFieldTypeDefault())
-	if err != nil {
-		return err
-	}
-	f.Field.idx = idx
-
-	if err := f.Open(); err != nil {
+	if err := f.parent.Open(false); err != nil {
+		f.parent = nil
 		return err
 	}
+	f.Field = f.parent.Field(name)
 	return nil
 }
 

field_test.go

@@ -15,7 +15,6 @@
 package pilosa_test
 
 import (
-	"io/ioutil"
 	"math"
 	"testing"
 
@@ -23,6 +22,7 @@ import (
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 var panicOn = pilosa.PanicOn
@@ -30,7 +30,7 @@ var panicOn = pilosa.PanicOn
 // Ensure a field can set & read a bsiGroup value.
 func TestField_SetValue(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
@@ -67,7 +67,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("Overwrite", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
@@ -103,7 +103,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("ErrBSIGroupNotFound", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
@@ -121,7 +121,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
@@ -139,7 +139,7 @@ func TestField_SetValue(t *testing.T) {
 	})
 
 	t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30))
@@ -158,7 +158,7 @@ func TestField_SetValue(t *testing.T) {
 }
 
 func TestField_NameRestriction(t *testing.T) {
-	path, err := ioutil.TempDir("", "pilosa-field-")
+	path, err := testhook.TempDir(t, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
@@ -190,7 +190,7 @@ func TestField_NameValidation(t *testing.T) {
 		"charact23112345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901",
 	}
 
-	path, err := ioutil.TempDir("", "pilosa-field-")
+	path, err := testhook.TempDir(t, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
@@ -210,7 +210,7 @@ func TestField_NameValidation(t *testing.T) {
 
 // Ensure can update and delete available shards.
 func TestField_AvailableShards(t *testing.T) {
-	idx := test.MustOpenIndex()
+	idx := test.MustOpenIndex(t)
 	defer idx.Close()
 
 	f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
@@ -253,7 +253,7 @@ func TestField_AvailableShards(t *testing.T) {
 
 func TestField_ClearValue(t *testing.T) {
 	t.Run("OK", func(t *testing.T) {
-		idx := test.MustOpenIndex()
+		idx := test.MustOpenIndex(t)
 		defer idx.Close()
 
 		f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))

fragment.go

@@ -45,6 +45,7 @@ import (
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/shardwidth"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 )
@@ -170,15 +171,6 @@ type fragment struct {
 	stats stats.StatsClient
 
 	bitmapInfo *roaring.BitmapInfo
-
-	// txTestingOnly: this looks gross.
-	// Nonetheless, it allowed us to
-	// integrate Tx into the
-	// fragment_internal_test.go suite
-	// and not break the world all at once.
-	//
-	// Only for testing, obviously.
-	txTestingOnly Tx
 }
 
 // newFragment returns a new instance of Fragment.
@@ -267,6 +259,7 @@ func (f *fragment) Open() error {
 	}
 	f.open = true
 
+	_ = testhook.Opened(f.holder.Auditor, f, nil)
 	f.holder.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard)
 	return nil
 }
@@ -513,6 +506,9 @@ func (f *fragment) openCache() error {
 func (f *fragment) Close() error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	defer func() {
+		_ = testhook.Closed(f.holder.Auditor, f, nil)
+	}()
 	for f.snapshotPending {
 		f.snapshotCond.Wait()
 	}

gendebug_test.go

@@ -17,23 +17,31 @@
 package pilosa
 
 import (
+	"errors"
 	"fmt"
-	"os"
-	"testing"
+	"runtime"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
-func examineResults() {
-	results := reportGenerations()
+func examineResults() error {
+	runtime.GC()
+	stats, results := reportGenerations()
+	if len(stats) > 0 {
+		fmt.Printf("generation stats: %s\n", stats)
+	}
+	if len(results) == 0 {
+		return nil
+	}
 	if len(results) > 0 {
 		fmt.Printf("generations:\n")
 		for _, res := range results {
 			fmt.Printf("  %s\n", res)
 		}
 	}
+	return errors.New("outstanding generations detected")
 }
 
-func TestMain(m *testing.M) {
-	ret := m.Run()
-	examineResults()
-	os.Exit(ret)
+func init() {
+	testhook.RegisterPostTestHook(examineResults)
 }

generation_debug.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"math/rand"
 	"runtime"
+	"runtime/debug"
 	"sort"
 	"sync"
 	"time"
@@ -29,6 +30,7 @@ const generationDebug = true
 
 type lifespan struct {
 	from, to, finalized time.Time
+	stack               []byte
 }
 
 var knownGenerations map[string]lifespan
@@ -36,32 +38,49 @@ var knownGenerationLock sync.Mutex
 
 var timeZero time.Time
 
+var generationDebugVerbose bool
+
+// History reports the finalized/dead/created status of a span which we think
+// is in some way in error. It's shared between a couple of places.
+func (span *lifespan) History() string {
+	dead := "not dead"
+	finalized := "not finalized"
+	if span.finalized != timeZero {
+		finalized = fmt.Sprintf("finalized at %v", span.finalized)
+	}
+	if span.to != timeZero {
+		dead = fmt.Sprintf("dead at %v", span.to)
+	}
+	return fmt.Sprintf("%s, %s, created at %v at %s", dead, finalized, span.from, span.stack)
+}
+
+func (span *lifespan) reportHistory(reason string, id string) string {
+	return fmt.Sprintf("%s %s: %s", id, reason, span.History())
+}
+
 func registerGeneration(id string) string {
 	knownGenerationLock.Lock()
 	defer knownGenerationLock.Unlock()
 	if knownGenerations == nil {
 		knownGenerations = make(map[string]lifespan)
 	}
-	newSpan := lifespan{from: time.Now()}
+	newSpan := lifespan{from: time.Now(), stack: debug.Stack()}
 	origId := id
 
 	// if you have more than 65k of the same file open, maybe you have bigger
 	// problems than this.
 	for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] {
 		suffix := fmt.Sprintf("::%04x", rand.Int63n(65536))
-		if span.finalized != timeZero {
-			fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v, finalized %v\n",
-				id, suffix, span.from, span.to, span.finalized)
-		} else {
-			if span.to != timeZero {
-				fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v\n", id, suffix, span.from, span.to)
-			} else {
-				fmt.Printf("new generation %s: adding %s, already exists, created %v", id, suffix, span.from)
-			}
+		if generationDebugVerbose {
+			history := span.History()
+			fmt.Printf("new generation: adding suffix %s, previous %s\n",
+				suffix, history)
 		}
 		id = origId + suffix
 	}
-	fmt.Printf("new generation %s\n", id)
+	if generationDebugVerbose {
+		fmt.Printf("new generation %s\n", id)
+	}
 	knownGenerations[id] = newSpan
 	return id
 }
@@ -75,8 +94,7 @@ func endGeneration(id string) {
 		panic(oops)
 	}
 	if span.finalized != timeZero || span.to != timeZero {
-		oops := fmt.Sprintf("ending generation %s: already died at %v, finalized at %v", id, span.to, span.finalized)
-		panic(oops)
+		panic(span.reportHistory("ending generation", id))
 	}
 	span.to = time.Now()
 	knownGenerations[id] = span
@@ -108,39 +126,25 @@ func finalizeGeneration(id string) {
 		panic(oops)
 	}
 	if span.finalized != timeZero {
-		var oops string
-		if span.to != timeZero {
-			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, but not dead", id, span.finalized)
-		} else {
-			oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, dead at %v", id, span.finalized, span.to)
-		}
-		panic(oops)
+		panic(span.reportHistory("finalizing", id))
 	}
 	span.finalized = time.Now()
 	knownGenerations[id] = span
 }
 
-func reportGenerations() []string {
+func reportGenerations() (stats string, surviving []string) {
 	runtime.GC()
 	knownGenerationLock.Lock()
 	defer knownGenerationLock.Unlock()
-	var surviving []string
 	times := make([]int64, 0, len(knownGenerations))
 	for id, span := range knownGenerations {
-		if span.to == timeZero {
-			if span.finalized == timeZero {
-				surviving = append(surviving, fmt.Sprintf("%s: %v, not ended or finalized", id, span.from))
-			} else {
-				surviving = append(surviving, fmt.Sprintf("%s: %v, finalized %v, not ended", id, span.from, span.finalized))
-			}
+		if span.to == timeZero || span.finalized == timeZero {
+			surviving = append(surviving, span.reportHistory("surviving", id))
 		} else {
-			if span.finalized == timeZero {
-				surviving = append(surviving, fmt.Sprintf("%s: %v to %v, not finalized", id, span.from, span.to))
-			} else {
-				times = append(times, int64(span.finalized.Sub(span.to)))
-			}
+			times = append(times, int64(span.finalized.Sub(span.to)))
 		}
 	}
+	stats = "no recorded finalized spans"
 	if len(times) > 0 {
 		sort.Slice(times, func(i, j int) bool { return times[i] < times[j] })
 		var total int64
@@ -153,8 +157,8 @@ func reportGenerations() []string {
 		p90 = times[(len(times)*9)/10]
 		p99 = times[(len(times)*99)/100]
 		worst = times[len(times)-1]
-		surviving = append(surviving, fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
-			len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst)))
+		stats = fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v",
+			len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst))
 	}
-	return surviving
+	return stats, surviving
 }

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
-	github.com/glycerine/lmdb-go v1.9.26
+	github.com/glycerine/lmdb-go v1.9.27
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/gogo/protobuf v1.2.0
 	github.com/golang/protobuf v1.3.3

go.sum

@@ -55,8 +55,8 @@ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy
 github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E=
 github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y=
-github.com/glycerine/lmdb-go v1.9.26 h1:4aIiCQhg5fLChuZuATDHD4Lr6y9CdEHLtvROkzCZKIg=
-github.com/glycerine/lmdb-go v1.9.26/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
+github.com/glycerine/lmdb-go v1.9.27 h1:k20zfiumwC/E1g/MYIzZ2GkhOH0hicDfEXa0KUjWLjY=
+github.com/glycerine/lmdb-go v1.9.27/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=

gossip/gossip.go

@@ -104,6 +104,7 @@ func (g *memberSet) Open() (err error) {
 // Close attempts to gracefully leave the cluster, and finally calls shutdown
 // after (at most) a timeout period.
 func (g *memberSet) Close() error {
+	g.eventReceiver.Close()
 	leaveErr := g.memberlist.Leave(5 * time.Second)
 	shutdownErr := g.memberlist.Shutdown()
 	if leaveErr != nil || shutdownErr != nil {
@@ -366,8 +367,9 @@ func (g *memberSet) MergeRemoteState(buf []byte, join bool) {
 // Care must be taken that events are processed in a timely manner from
 // the channel, since this delegate will block until an event can be sent.
 type eventReceiver struct {
-	ch   chan memberlist.NodeEvent
-	papi *pilosa.API
+	ch     chan memberlist.NodeEvent
+	closed chan struct{}
+	papi   *pilosa.API
 
 	logger logger.Logger
 }
@@ -376,6 +378,7 @@ type eventReceiver struct {
 func newEventReceiver(logger logger.Logger, papi *pilosa.API) *eventReceiver {
 	ger := &eventReceiver{
 		ch:     make(chan memberlist.NodeEvent, 1),
+		closed: make(chan struct{}),
 		logger: logger,
 		papi:   papi,
 	}
@@ -389,7 +392,10 @@ func (g *eventReceiver) NotifyJoin(n *memberlist.Node) {
 	n2.Meta = make([]byte, len(n.Meta))
 	copy(n2.Meta, n.Meta)
 
-	g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}
+	select {
+	case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}:
+	case <-g.closed:
+	}
 }
 
 func (g *eventReceiver) NotifyLeave(n *memberlist.Node) {
@@ -398,7 +404,10 @@ func (g *eventReceiver) NotifyLeave(n *memberlist.Node) {
 	n2.Meta = make([]byte, len(n.Meta))
 	copy(n2.Meta, n.Meta)
 
-	g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}
+	select {
+	case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}:
+	case <-g.closed:
+	}
 }
 
 func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) {
@@ -407,13 +416,25 @@ func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) {
 	n2.Meta = make([]byte, len(n.Meta))
 	copy(n2.Meta, n.Meta)
 
-	g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}
+	select {
+	case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}:
+	case <-g.closed:
+	}
+}
+
+func (g *eventReceiver) Close() {
+	close(g.closed)
 }
 
 func (g *eventReceiver) listen() {
 	var nodeEventType pilosa.NodeEventType
 	for {
-		e := <-g.ch
+		var e memberlist.NodeEvent
+		select {
+		case <-g.closed:
+			return
+		case e = <-g.ch:
+		}
 		switch e.Event {
 		case memberlist.NodeJoin:
 			nodeEventType = pilosa.NodeJoin

holder.go

@@ -32,6 +32,7 @@ import (
 	"github.com/pilosa/pilosa/v2/logger"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pilosa/pilosa/v2/tracing"
 	"github.com/pkg/errors"
 	uuid "github.com/satori/go.uuid"
@@ -105,6 +106,8 @@ type Holder struct {
 	opening bool
 
 	Opts HolderOpts
+
+	Auditor testhook.Auditor
 }
 
 type HolderOpts struct {
@@ -165,7 +168,7 @@ func (lc *lockedChan) Recv() {
 
 // NewHolder returns a new instance of Holder.
 func NewHolder(partitionN int) *Holder {
-	return &Holder{
+	h := &Holder{
 		partitionN: partitionN,
 		indexes:    make(map[string]*Index),
 		closing:    make(chan struct{}),
@@ -188,7 +191,11 @@ func NewHolder(partitionN int) *Holder {
 		Logger: logger.NopLogger,
 
 		SnapshotQueue: defaultSnapshotQueue,
+
+		Auditor: NewAuditor(),
 	}
+	_ = testhook.Created(h.Auditor, h, nil)
+	return h
 }
 
 type HolderInfo struct {
@@ -535,6 +542,8 @@ func (h *Holder) Open() error {
 			err = index.Open(false)
 		}
 		if err != nil {
+			// FIXME: The holder shouldn't be responsible for closing these, probably.
+			_ = index.Txf.CloseDB()
 			if err == ErrName {
 				h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err)
 				continue
@@ -559,6 +568,8 @@ func (h *Holder) Open() error {
 
 	h.opened.Close()
 
+	_ = testhook.Opened(h.Auditor, h, nil)
+
 	return nil
 }
 
@@ -627,6 +638,12 @@ func (h *Holder) Close() error {
 	h.opened.mu.Lock()
 	h.opened.ch = make(chan struct{})
 	h.opened.mu.Unlock()
+	if h.SnapshotQueue != nil {
+		h.SnapshotQueue.Stop()
+		h.SnapshotQueue = nil
+	}
+
+	_ = testhook.Closed(h.Auditor, h, nil)
 
 	return nil
 }

holder_internal_test.go

@@ -16,9 +16,10 @@ package pilosa
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 type testHolderOperator struct {
@@ -72,8 +73,8 @@ func (t *testHolderOperator) ProcessFragment(*fragment) error {
 	return nil
 }
 
-func makeHolder() (*Holder, string, error) {
-	path, err := ioutil.TempDir("", "pilosa-")
+func makeHolder(tb testing.TB) (*Holder, string, error) {
+	path, err := testhook.TempDir(tb, "pilosa-")
 	if err != nil {
 		return nil, "", err
 	}
@@ -109,7 +110,7 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui
 }
 
 func TestHolderOperatorProcess(t *testing.T) {
-	h, path, err := makeHolder()
+	h, path, err := makeHolder(t)
 	if err != nil {
 		t.Fatalf("creating holder: %v", err)
 	}
@@ -139,7 +140,7 @@ func TestHolderOperatorProcess(t *testing.T) {
 }
 
 func TestHolderOperatorCancel(t *testing.T) {
-	h, path, err := makeHolder()
+	h, path, err := makeHolder(t)
 	if err != nil {
 		t.Fatalf("creating holder: %v", err)
 	}

holder_test.go

@@ -33,7 +33,7 @@ import (
 
 func TestHolder_Open(t *testing.T) {
 	t.Run("ErrIndexName", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 
 		bufLogger := test.NewBufferLogger()
 		h.Holder.Logger = bufLogger
@@ -60,7 +60,7 @@ func TestHolder_Open(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
@@ -79,7 +79,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
@@ -99,7 +99,7 @@ func TestHolder_Open(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@@ -119,7 +119,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		var idx *pilosa.Index
@@ -142,7 +142,7 @@ func TestHolder_Open(t *testing.T) {
 		}
 	})
 	t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		var idx *pilosa.Index
@@ -170,7 +170,7 @@ func TestHolder_Open(t *testing.T) {
 		if os.Geteuid() == 0 {
 			t.Skip("Skipping permissions test since user is root.")
 		}
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		var idx *pilosa.Index
@@ -205,7 +205,7 @@ func TestHolder_Open(t *testing.T) {
 	t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
 		roaringOnlyTest(t)
 
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		var idx *pilosa.Index
@@ -239,7 +239,7 @@ func TestHolder_Open(t *testing.T) {
 	t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
 		roaringOnlyTest(t)
 
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		idx, err := h.CreateIndex("foo", pilosa.IndexOptions{})
@@ -271,7 +271,7 @@ func TestHolder_Open(t *testing.T) {
 
 	t.Run("ForeignIndex", func(t *testing.T) {
 		t.Run("ErrForeignIndexNotFound", func(t *testing.T) {
-			h := test.MustOpenHolder()
+			h := test.MustOpenHolder(t)
 			defer h.Close()
 
 			if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@@ -288,7 +288,7 @@ func TestHolder_Open(t *testing.T) {
 
 		// Foreign index zzz is opened after foo/bar.
 		t.Run("ForeignIndexNotOpenYet", func(t *testing.T) {
-			h := test.MustOpenHolder()
+			h := test.MustOpenHolder(t)
 			defer h.Close()
 
 			if _, err := h.CreateIndex("zzz", pilosa.IndexOptions{}); err != nil {
@@ -308,7 +308,7 @@ func TestHolder_Open(t *testing.T) {
 
 		// Foreign index aaa is opened before foo/bar.
 		t.Run("ForeignIndexIsOpen", func(t *testing.T) {
-			h := test.MustOpenHolder()
+			h := test.MustOpenHolder(t)
 			defer h.Close()
 
 			if _, err := h.CreateIndex("aaa", pilosa.IndexOptions{}); err != nil {
@@ -328,7 +328,7 @@ func TestHolder_Open(t *testing.T) {
 
 		// Try to re-create existing index
 		t.Run("CreateIndexIfNotExists", func(t *testing.T) {
-			h := test.MustOpenHolder()
+			h := test.MustOpenHolder(t)
 			defer h.Close()
 
 			idx1, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{})
@@ -356,7 +356,7 @@ func TestHolder_Open(t *testing.T) {
 
 func TestHolder_HasData(t *testing.T) {
 	t.Run("IndexDirectory", func(t *testing.T) {
-		h := test.MustOpenHolder()
+		h := test.MustOpenHolder(t)
 		defer h.Close()
 
 		if ok, err := h.HasData(); ok || err != nil {
@@ -373,7 +373,7 @@ func TestHolder_HasData(t *testing.T) {
 	})
 
 	t.Run("Peek", func(t *testing.T) {
-		h := test.NewHolder()
+		h := test.NewHolder(t)
 
 		if ok, err := h.HasData(); ok || err != nil {
 			t.Fatal("expected HasData to return false, no err, but", ok, err)
@@ -390,7 +390,7 @@ func TestHolder_HasData(t *testing.T) {
 	})
 
 	t.Run("Peek at missing directory", func(t *testing.T) {
-		h := test.NewHolder()
+		h := test.NewHolder(t)
 
 		// Ensure that hasData is false when dir doesn't exist.
 		h.Path = "bad-path"
@@ -404,7 +404,7 @@ func TestHolder_HasData(t *testing.T) {
 // Ensure holder can delete an index and its underlying files.
 func TestHolder_DeleteIndex(t *testing.T) {
 
-	hldr := test.MustOpenHolder()
+	hldr := test.MustOpenHolder(t)
 	defer hldr.Close()
 
 	// Write bits to separate indexes.
@@ -432,43 +432,43 @@ func TestHolder_DeleteIndex(t *testing.T) {
 // Ensure holder can sync with a remote holder.
 func TestHolderSyncer_SyncHolder(t *testing.T) {
 	c := test.MustNewCluster(t, 2)
-	c[0].Config.Cluster.ReplicaN = 2
-	c[0].Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 2
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.Cluster.ReplicaN = 2
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 2
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
 	}
 	defer c.Close()
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index y: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f0: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field z in y: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool())
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool())
 	if err != nil {
 		t.Fatalf("creating field b in y: %v", err)
 	}
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 	// Set data on the local holder.
 	hldr0.SetBit("i", "f", 0, 10)
@@ -495,11 +495,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 	hldr1.SetBit("y", "b", 0, (3*ShardWidth)+5) // false
 	hldr1.SetBit("y", "b", 1, (3*ShardWidth)+7) // true
 
-	err = c[0].Server.SyncData()
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
-	err = c[1].Server.SyncData()
+	err = c.GetNode(1).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 1: %v", err)
 	}
@@ -543,30 +543,30 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
 // the row boundaries of the block.
 func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
 	c := test.MustNewCluster(t, 3)
-	c[0].Config.Cluster.ReplicaN = 3
-	c[0].Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 3
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.Cluster.ReplicaN = 3
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 3
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
 	}
 	defer c.Close()
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
 
 	blockEdge := uint64(pilosa.HashBlockSize)
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
-	hldr2 := &test.Holder{Holder: c[2].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
+	hldr2 := &test.Holder{Holder: c.GetNode(2).Server.Holder()}
 
 	// Set data on the local holder.
 	hldr0.SetBit("i", "f", blockEdge-1, 10)
@@ -579,7 +579,7 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
 
 	// Leave the third replica empty to force a block merge.
 	//
-	err = c[0].Server.SyncData()
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
@@ -598,28 +598,28 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
 // Ensure holder correctly handles clears during block sync.
 func TestHolderSyncer_Clears(t *testing.T) {
 	c := test.MustNewCluster(t, 3)
-	c[0].Config.Cluster.ReplicaN = 3
-	c[0].Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 3
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.Cluster.ReplicaN = 3
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 3
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
 	}
 	defer c.Close()
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
-	hldr2 := &test.Holder{Holder: c[2].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
+	hldr2 := &test.Holder{Holder: c.GetNode(2).Server.Holder()}
 
 	// Set data on the local holder that should be cleared
 	// because it's the only instance of this value.
@@ -631,7 +631,7 @@ func TestHolderSyncer_Clears(t *testing.T) {
 	hldr1.SetBit("i", "f", 0, 20)
 	hldr2.SetBit("i", "f", 0, 20)
 
-	err = c[0].Server.SyncData()
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
@@ -647,10 +647,10 @@ func TestHolderSyncer_Clears(t *testing.T) {
 // Ensure holder can sync time quantum views with a remote holder.
 func TestHolderSyncer_TimeQuantum(t *testing.T) {
 	c := test.MustNewCluster(t, 2)
-	c[0].Config.Cluster.ReplicaN = 2
-	c[0].Config.AntiEntropy.Interval = 0
-	c[1].Config.Cluster.ReplicaN = 2
-	c[1].Config.AntiEntropy.Interval = 0
+	c.GetNode(0).Config.Cluster.ReplicaN = 2
+	c.GetNode(0).Config.AntiEntropy.Interval = 0
+	c.GetNode(1).Config.Cluster.ReplicaN = 2
+	c.GetNode(1).Config.AntiEntropy.Interval = 0
 	err := c.Start()
 	if err != nil {
 		t.Fatalf("starting cluster: %v", err)
@@ -659,17 +659,17 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 
 	quantum := "D"
 
-	_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index i: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum)))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum)))
 	if err != nil {
 		t.Fatalf("creating field f: %v", err)
 	}
 
-	hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-	hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+	hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+	hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 	// Set data on the local holder for node0.
 	t1 := time.Date(2018, 8, 1, 12, 30, 0, 0, time.UTC)
@@ -680,7 +680,7 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 	// Set data on node1.
 	hldr1.SetBitTime("i", "f", 0, 22, &t2)
 
-	err = c[0].Server.SyncData()
+	err = c.GetNode(0).Server.SyncData()
 	if err != nil {
 		t.Fatalf("syncing node 0: %v", err)
 	}
@@ -700,10 +700,10 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
 func TestHolderSyncer_IntField(t *testing.T) {
 	t.Run("BasicSync", func(t *testing.T) {
 		c := test.MustNewCluster(t, 2)
-		c[0].Config.Cluster.ReplicaN = 2
-		c[0].Config.AntiEntropy.Interval = 0
-		c[1].Config.Cluster.ReplicaN = 2
-		c[1].Config.AntiEntropy.Interval = 0
+		c.GetNode(0).Config.Cluster.ReplicaN = 2
+		c.GetNode(0).Config.AntiEntropy.Interval = 0
+		c.GetNode(1).Config.Cluster.ReplicaN = 2
+		c.GetNode(1).Config.AntiEntropy.Interval = 0
 		err := c.Start()
 		if err != nil {
 			t.Fatalf("starting cluster: %v", err)
@@ -712,18 +712,18 @@ func TestHolderSyncer_IntField(t *testing.T) {
 
 		var idx0 *pilosa.Index
 		_ = idx0
-		idx0, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 		_ = idx0
 		if err != nil {
 			t.Fatalf("creating index i: %v", err)
 		}
-		_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100))
+		_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100))
 		if err != nil {
 			t.Fatalf("creating field f: %v", err)
 		}
 
-		hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-		hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+		hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+		hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 		// Set data on the local holder for node0. columnID=1, value=1
 		hldr0.SetValue("i", "f", 1, 1)
@@ -734,7 +734,7 @@ func TestHolderSyncer_IntField(t *testing.T) {
 		idx1 := hldr1.SetValue("i", "f", 2, 2)
 		_ = idx1
 
-		err = c[0].Server.SyncData()
+		err = c.GetNode(0).Server.SyncData()
 		if err != nil {
 			t.Fatalf("syncing node 0: %v", err)
 		}
@@ -758,10 +758,10 @@ func TestHolderSyncer_IntField(t *testing.T) {
 
 	t.Run("MultiShard", func(t *testing.T) {
 		c := test.MustNewCluster(t, 2)
-		c[0].Config.Cluster.ReplicaN = 2
-		c[0].Config.AntiEntropy.Interval = 0
-		c[1].Config.Cluster.ReplicaN = 2
-		c[1].Config.AntiEntropy.Interval = 0
+		c.GetNode(0).Config.Cluster.ReplicaN = 2
+		c.GetNode(0).Config.AntiEntropy.Interval = 0
+		c.GetNode(1).Config.Cluster.ReplicaN = 2
+		c.GetNode(1).Config.AntiEntropy.Interval = 0
 		err := c.Start()
 		if err != nil {
 			t.Fatalf("starting cluster: %v", err)
@@ -770,18 +770,18 @@ func TestHolderSyncer_IntField(t *testing.T) {
 
 		var idx0 *pilosa.Index
 		_ = idx0
-		idx0, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+		idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 		_ = idx0
 		if err != nil {
 			t.Fatalf("creating index i: %v", err)
 		}
-		_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
+		_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
 		if err != nil {
 			t.Fatalf("creating field f: %v", err)
 		}
 
-		hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
-		hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
+		hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()}
+		hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()}
 
 		// Set data on the local holder for node0.
 		hldr0.SetValue("i", "f", 1*pilosa.ShardWidth, 11)
@@ -799,11 +799,11 @@ func TestHolderSyncer_IntField(t *testing.T) {
 		// node0: [0,3,7]
 		// node1: [1,2,4]
 
-		err = c[0].Server.SyncData()
+		err = c.GetNode(0).Server.SyncData()
 		if err != nil {
 			t.Fatalf("syncing node 0: %v", err)
 		}
-		err = c[1].Server.SyncData()
+		err = c.GetNode(1).Server.SyncData()
 		if err != nil {
 			t.Fatalf("syncing node 1: %v", err)
 		}

http/client_test.go

@@ -49,7 +49,7 @@ func TestClient_MultiNode(t *testing.T) {
 	defer c.Close()
 
 	hldr := []test.Holder{}
-	for _, command := range c {
+	for _, command := range c.Nodes {
 		hldr = append(hldr, test.Holder{Holder: command.Server.Holder()})
 	}
 
@@ -72,7 +72,7 @@ func TestClient_MultiNode(t *testing.T) {
 			}
 		}
 		if !ownsNum {
-			t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c[i].URL(), owns)
+			t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c.GetNode(i).URL(), owns)
 		}
 	}
 
@@ -86,11 +86,11 @@ func TestClient_MultiNode(t *testing.T) {
 			maxShard = x
 		}
 	}
-	_, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err := c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
-	_, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
+	_, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
 	if err != nil {
 		t.Fatalf("creating field: %v", err)
 	}
@@ -119,24 +119,24 @@ func TestClient_MultiNode(t *testing.T) {
 	// Rebuild the RankCache.
 	// We have to do this to avoid the 10-second cache invalidation delay
 	// built into cache.Invalidate()
-	err = c[0].RecalculateCaches(t)
+	err = c.GetNode(0).RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
-	err = c[1].RecalculateCaches(t)
+	err = c.GetNode(1).RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
-	err = c[2].RecalculateCaches(t)
+	err = c.GetNode(2).RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating cache: %v", err)
 	}
 
 	// Connect to each node to compare results.
 	client := make([]*Client, 3)
-	client[0] = MustNewClient(c[0].URL(), http.GetHTTPClient(nil))
-	client[1] = MustNewClient(c[1].URL(), http.GetHTTPClient(nil))
-	client[2] = MustNewClient(c[2].URL(), http.GetHTTPClient(nil))
+	client[0] = MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil))
+	client[1] = MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil))
+	client[2] = MustNewClient(c.GetNode(2).URL(), http.GetHTTPClient(nil))
 
 	topN := 4
 	queryRequest := &pilosa.QueryRequest{
@@ -188,7 +188,7 @@ func TestClient_MultiNode(t *testing.T) {
 func TestClient_Export(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	host := cmd.URL()
 
@@ -368,7 +368,7 @@ func TestClient_Export(t *testing.T) {
 func TestClient_Import(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	host := cmd.URL()
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -415,7 +415,7 @@ func TestClient_Import(t *testing.T) {
 // Ensure client can bulk import column attrs.
 func TestClient_ImportColumnAttrs(t *testing.T) {
 	cluster := test.MustNewCluster(t, 2)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -425,31 +425,31 @@ func TestClient_ImportColumnAttrs(t *testing.T) {
 	defer cluster.Close()
 
 	ctx := context.Background()
-	_, err = cluster[0].API.CreateIndex(ctx, "i", pilosa.IndexOptions{})
+	_, err = cluster.GetNode(0).API.CreateIndex(ctx, "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
-	_, err = cluster[0].API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
+	_, err = cluster.GetNode(0).API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
 	if err != nil {
 		t.Fatalf("creating field: %v", err)
 	}
-	_, err = cluster[0].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"})
+	_, err = cluster.GetNode(0).API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"})
 	if err != nil {
 		t.Fatalf("querying: %v", err)
 	}
 
 	attrKey := "k"
 	// Send import request.
-	host := cluster[0].URL()
+	host := cluster.GetNode(0).URL()
 	c := MustNewClient(host, http.GetHTTPClient(nil))
 	colAttrsReq := makeImportColumnAttrsRequest("i", 0, attrKey)
-	if err := c.ImportColumnAttrs(ctx, &cluster[1].API.Node().URI, "i", colAttrsReq); err != nil {
+	if err := c.ImportColumnAttrs(ctx, &cluster.GetNode(1).API.Node().URI, "i", colAttrsReq); err != nil {
 		t.Fatal(err)
 	}
 
 	// Verify data.
 	pql := "Options(Row(f=0), columnAttrs=true)"
-	res, err := cluster[1].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql})
+	res, err := cluster.GetNode(1).API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -469,7 +469,7 @@ func TestClient_ImportColumnAttrs(t *testing.T) {
 // Ensure client can bulk import data.
 func TestClient_ImportRoaring(t *testing.T) {
 	cluster := test.MustNewCluster(t, 2)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -478,29 +478,29 @@ func TestClient_ImportRoaring(t *testing.T) {
 	}
 	defer cluster.Close()
 
-	_, err = cluster[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
+	_, err = cluster.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
 	if err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
-	_, err = cluster[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
+	_, err = cluster.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100))
 	if err != nil {
 		t.Fatalf("creating field: %v", err)
 	}
-	_, err = cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"})
+	_, err = cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"})
 	if err != nil {
 		t.Fatalf("querying: %v", err)
 	}
 
 	// Send import request.
-	host := cluster[0].URL()
+	host := cluster.GetNode(0).URL()
 	c := MustNewClient(host, http.GetHTTPClient(nil))
 	// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
 	roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100")
-	if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
 		t.Fatal(err)
 	}
 
-	hldr := test.Holder{Holder: cluster[0].Server.Holder()}
+	hldr := test.Holder{Holder: cluster.GetNode(0).Server.Holder()}
 	// Verify data on node 0.
 	if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
 		t.Fatalf("unexpected columns: %+v", a)
@@ -509,7 +509,7 @@ func TestClient_ImportRoaring(t *testing.T) {
 		t.Fatalf("unexpected columns: %+v", a)
 	}
 
-	hldr2 := test.Holder{Holder: cluster[1].Server.Holder()}
+	hldr2 := test.Holder{Holder: cluster.GetNode(1).Server.Holder()}
 	// Verify data on node 1.
 	if a := hldr2.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) {
 		t.Fatalf("unexpected columns: %+v", a)
@@ -521,7 +521,7 @@ func TestClient_ImportRoaring(t *testing.T) {
 	// Ensure that sending a roaring import with the clear flag works as expected.
 	// [65539, 65540]
 	roaringReq = makeImportRoaringRequest(true, "3A30000001000000010001001000000003000400")
-	if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
 		t.Fatal(err)
 	}
 
@@ -544,7 +544,7 @@ func TestClient_ImportRoaring(t *testing.T) {
 	// Ensure that sending a roaring import with the clear flag works as expected.
 	// [4, 6, 65537, 65539]
 	roaringReq = makeImportRoaringRequest(true, "3A300000020000000000010001000100180000001C0000000400060001000300")
-	if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
 		t.Fatal(err)
 	}
 
@@ -567,7 +567,7 @@ func TestClient_ImportRoaring(t *testing.T) {
 	// Ensure that sending a roaring import with the clear flag works as expected.
 	// [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537]
 	roaringReq = makeImportRoaringRequest(true, "3B3001000100000900010000000100010009000100")
-	if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
+	if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil {
 		t.Fatal(err)
 	}
 
@@ -593,7 +593,7 @@ func TestClient_ImportKeys(t *testing.T) {
 	t.Run("SingleNode", func(t *testing.T) {
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		host := cmd.URL()
 
 		cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true})
@@ -691,8 +691,8 @@ func TestClient_ImportKeys(t *testing.T) {
 	t.Run("MultiNode", func(t *testing.T) {
 		cluster := test.MustRunCluster(t, 2)
 		defer cluster.Close()
-		cmd0 := cluster[0]
-		cmd1 := cluster[1]
+		cmd0 := cluster.GetNode(0)
+		cmd1 := cluster.GetNode(1)
 		host0 := cmd0.URL()
 		host1 := cmd1.URL()
 
@@ -768,7 +768,7 @@ func TestClient_ImportKeys(t *testing.T) {
 	t.Run("IntegerFieldSingleNode", func(t *testing.T) {
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		host := cmd.URL()
 		holder := cmd.Server.Holder()
 		hldr := test.Holder{Holder: holder}
@@ -840,7 +840,7 @@ func TestClient_ImportIDs(t *testing.T) {
 	t.Run("ImportRangeImport", func(t *testing.T) {
 		cluster := test.MustRunCluster(t, 1)
 		defer cluster.Close()
-		cmd := cluster[0]
+		cmd := cluster.GetNode(0)
 		host := cmd.URL()
 		holder := cmd.Server.Holder()
 		hldr := test.Holder{Holder: holder}
@@ -901,7 +901,7 @@ func TestClient_ImportIDs(t *testing.T) {
 func TestClient_ImportValue(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	host := cmd.URL()
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -974,7 +974,7 @@ func TestClient_ImportValue(t *testing.T) {
 func TestClient_ImportExistence(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	host := cmd.URL()
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -1053,7 +1053,7 @@ func TestClient_ImportExistence(t *testing.T) {
 func TestClient_FragmentBlocks(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -1086,7 +1086,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
 func TestClient_CreateDecimalField(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil))
 
@@ -1195,8 +1195,8 @@ func TestClientTransactions(t *testing.T) {
 	c := test.MustRunCluster(t, 3)
 	defer c.Close()
 
-	client0 := MustNewClient(c[0].URL(), http.GetHTTPClient(nil))
-	client1 := MustNewClient(c[1].URL(), http.GetHTTPClient(nil))
+	client0 := MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil))
+	client1 := MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil))
 
 	// can create, list, get, and finish a transaction
 	var expDeadline time.Time

http/translator_test.go

@@ -36,7 +36,7 @@ func TestTranslateStore_EntryReader(t *testing.T) {
 
 			cluster := test.MustRunCluster(t, 1)
 			defer cluster.Close()
-			primary := cluster[0]
+			primary := cluster.GetNode(0)
 
 			hldr := test.Holder{Holder: primary.Server.Holder()}
 			index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
@@ -157,7 +157,7 @@ func benchmarkSetup(b *testing.B, ctx context.Context, key string, nkeys int) (s
 	b.Helper()
 
 	cluster := test.MustRunCluster(b, 1)
-	primary := cluster[0]
+	primary := cluster.GetNode(0)
 
 	idx := primary.MustCreateIndex(b, "i", pilosa.IndexOptions{})
 	fld := primary.MustCreateField(b, idx.Name(), "f", pilosa.OptFieldKeys())

index.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pilosa/pilosa/v2/internal"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
@@ -153,6 +154,9 @@ func (i *Index) CreatedAt() int64 {
 // Name returns name of the index.
 func (i *Index) Name() string { return i.name }
 
+// Holder yields this index's Holder.
+func (i *Index) Holder() *Holder { return i.holder }
+
 // QualifiedName returns the qualified name of the index.
 func (i *Index) QualifiedName() string { return i.qualifiedName }
 
@@ -253,6 +257,7 @@ func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) {
 		return err
 	}
 
+	_ = testhook.Opened(i.holder.Auditor, i, nil)
 	return nil
 }
 
@@ -339,7 +344,16 @@ fileLoop:
 			})
 		}
 	}
-	return eg.Wait()
+	err = eg.Wait()
+	if err != nil {
+		// Close any fields which got opened, since the overall
+		// index won't be open.
+		for n, f := range i.fields {
+			f.Close()
+			delete(i.fields, n)
+		}
+	}
+	return err
 }
 
 // openExistenceField gets or creates the existence field and associates it to the index.
@@ -403,6 +417,9 @@ func (i *Index) saveMeta() error {
 func (i *Index) Close() error {
 	i.mu.Lock()
 	defer i.mu.Unlock()
+	defer func() {
+		_ = testhook.Closed(i.holder.Auditor, i, nil)
+	}()
 
 	err := i.Txf.CloseIndex(i)
 	if err != nil {

index_internal_test.go

@@ -15,19 +15,23 @@
 package pilosa
 
 import (
-	"io/ioutil"
 	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // mustOpenIndex returns a new, opened index at a temporary path. Panic on error.
-func mustOpenIndex(opt IndexOptions) *Index {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-index-")
+func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}
 	h := NewHolder(1)
 	h.Path = path
 	index, err := h.CreateIndex("i", opt)
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 
 	if err != nil {
 		panic(err)
@@ -36,9 +40,6 @@ func mustOpenIndex(opt IndexOptions) *Index {
 	index.keys = opt.Keys
 	index.trackExistence = opt.TrackExistence
 
-	if err := index.Open(false); err != nil {
-		panic(err)
-	}
 	return index
 }
 
@@ -56,7 +57,7 @@ func (i *Index) reopen() error {
 // Ensure that deleting the existence field is handled properly.
 func TestIndex_Existence_Delete(t *testing.T) {
 	// Create Index (with existence tracking).
-	index := mustOpenIndex(IndexOptions{TrackExistence: true})
+	index := mustOpenIndex(t, IndexOptions{TrackExistence: true})
 	defer index.Close()
 
 	// Ensure existence field has been created.

index_test.go

@@ -15,13 +15,13 @@
 package pilosa_test
 
 import (
-	"io/ioutil"
 	"reflect"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/test"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -30,7 +30,7 @@ const ShardWidth = pilosa.ShardWidth
 
 // Ensure index can open and retrieve a field.
 func TestIndex_CreateFieldIfNotExists(t *testing.T) {
-	index := test.MustOpenIndex()
+	index := test.MustOpenIndex(t)
 	defer index.Close()
 
 	// Create field.
@@ -58,7 +58,7 @@ func TestIndex_CreateField(t *testing.T) {
 	// Ensure time quantum can be set appropriately on a new field.
 	t.Run("TimeQuantum", func(t *testing.T) {
 		t.Run("Explicit", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			// Create field with explicit quantum.
@@ -74,7 +74,7 @@ func TestIndex_CreateField(t *testing.T) {
 	// Ensure time quantum can be set appropriately on a new field.
 	t.Run("TimeQuantumNoStandardView", func(t *testing.T) {
 		t.Run("Explicit", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			// Create field with explicit quantum with no standard view
@@ -90,7 +90,7 @@ func TestIndex_CreateField(t *testing.T) {
 	// Ensure field can include range columns.
 	t.Run("BSIFields", func(t *testing.T) {
 		t.Run("OK", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			// Create field with schema and verify it exists.
@@ -112,7 +112,7 @@ func TestIndex_CreateField(t *testing.T) {
 		// on field creation FieldOptions validation.
 		/*
 			t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -123,7 +123,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
 					CacheType: pilosa.CacheTypeNone,
@@ -134,7 +134,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("ErrFieldFieldsAllowed", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -147,7 +147,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("ErrFieldNameRequired", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -160,7 +160,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("ErrInvalidFieldType", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -173,7 +173,7 @@ func TestIndex_CreateField(t *testing.T) {
 			})
 
 			t.Run("ErrInvalidBSIGroupRange", func(t *testing.T) {
-				index := test.MustOpenIndex()
+				index := test.MustOpenIndex(t)
 				defer index.Close()
 
 				if _, err := index.CreateField("f", pilosa.FieldOptions{
@@ -190,7 +190,7 @@ func TestIndex_CreateField(t *testing.T) {
 	t.Run("WithKeys", func(t *testing.T) {
 		// Don't allow an int field to be created with keys=true
 		t.Run("IntField", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			_, err := index.CreateField("f", pilosa.OptFieldTypeInt(-1, 1), pilosa.OptFieldKeys())
@@ -201,7 +201,7 @@ func TestIndex_CreateField(t *testing.T) {
 
 		// Don't allow a decimal field to be created with keys=true
 		t.Run("DecimalField", func(t *testing.T) {
-			index := test.MustOpenIndex()
+			index := test.MustOpenIndex(t)
 			defer index.Close()
 
 			_, err := index.CreateField("f", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -1}, pql.Decimal{Value: 1}), pilosa.OptFieldKeys())
@@ -214,7 +214,7 @@ func TestIndex_CreateField(t *testing.T) {
 
 // Ensure index can delete a field.
 func TestIndex_DeleteField(t *testing.T) {
-	index := test.MustOpenIndex()
+	index := test.MustOpenIndex(t)
 	defer index.Close()
 
 	// Create field.
@@ -238,7 +238,7 @@ func TestIndex_DeleteField(t *testing.T) {
 
 // Ensure index can validate its name.
 func TestIndex_InvalidName(t *testing.T) {
-	path, err := ioutil.TempDir("", "pilosa-index-")
+	path, err := testhook.TempDir(t, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}

lmdb.go

@@ -130,14 +130,15 @@ func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
 
 	err = env.SetMaxDBs(1)
 	panicOn(err)
-	err = env.SetMapSize(256 << 30) // 256GB
+	//err = env.SetMapSize(256 << 30) // 256GB
+	err = env.SetMapSize(16 << 30) // 16GB
 	panicOn(err)
 
 	panicOn(os.MkdirAll(filepath.Dir(path), 0755))
 
 	flags := uint(lmdb.NoReadahead | lmdb.NoSubdir)
 
-	// unsafe, but get upper bound on performance. TODO: remove these.
+	// unsafe, but get upper bound on performance.
 	//	WriteMap    = C.MDB_WRITEMAP   // Use a writable memory map.
 	//	NoMetaSync  = C.MDB_NOMETASYNC // Don't fsync metapage after commit.
 	//	NoSync      = C.MDB_NOSYNC     // Don't fsync after commit.
@@ -149,11 +150,18 @@ func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
 
 	// ACI not ACID at the moment; no durability
 	flags = flags |
-		//lmdb.WriteMap | // Use a writable memory map.
+		lmdb.NoMemInit | // Disable LMDB memory initialization
+
+		// Note that lmdb.WriteMap requests a big, writable, memory map.
+		// On my darwin/OSX laptop with 16GB ram, for instance, we
+		// can have difficulty obtaining this, resulting in
+		//   panic: mdb_env_open: no space left on device
+		lmdb.WriteMap | // Use a writable memory map.
+
+		// default ACI (not Durable) transactions; 300% faster write speed results.
 		lmdb.NoMetaSync | // Don't fsync metapage after commit.
 		lmdb.NoSync | // Don't fsync after commit.
-		lmdb.MapAsync | // Flush asynchronously when using the WriteMap flag.
-		lmdb.NoMemInit // Disable LMDB memory initialization
+		lmdb.MapAsync // Flush asynchronously when using the WriteMap flag.
 
 	err = env.Open(path, flags, 0644)
 	if err != nil {

main_test.go

@@ -0,0 +1,25 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+func TestMain(m *testing.M) {
+	testhook.RunTestsWithHooks(m)
+}

mmap_test.go

@@ -31,11 +31,12 @@ type cv struct {
 
 func forceSnapshotsCheckMapping(t *testing.T) {
 	depth := uint(6)
-	f, idx := mustOpenBSIFragment("i", "f", viewStandard, 0)
+	f, idx, tx := mustOpenBSIFragment(t, "i", "f", viewStandard, 0)
+	tx.Rollback()
 	f.Logger = logger.NewLogfLogger(t)
 	defer f.Clean(t)
 
-	tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
+	tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
 	defer tx.Rollback()
 
 	for i := 0; i < f.MaxOpN; i++ {

pg/server_test.go

@@ -35,7 +35,7 @@ func TestStartupTimeout(t *testing.T) {
 
 	connect, shutdown, err := pgtest.ServeMem(&pg.Server{
 		StartupTimeout: time.Millisecond,
-		Logger:         logger.NewLogfLogger(t),
+		Logger:         logger.NopLogger,
 	})
 	if err != nil {
 		t.Fatalf("starting in-memory postgres server: %v", err)
@@ -103,7 +103,7 @@ func TestPQConnect(t *testing.T) {
 
 	server := &pg.Server{
 		StartupTimeout: time.Second,
-		Logger:         logger.NewLogfLogger(t),
+		Logger:         logger.NopLogger,
 	}
 	addr, shutdown, err := pgtest.ServeTCP(":0", server)
 	if err != nil {
@@ -134,7 +134,7 @@ func TestPQConnectSSL(t *testing.T) {
 
 	server := &pg.Server{
 		StartupTimeout: time.Second,
-		Logger:         logger.NewLogfLogger(t),
+		Logger:         logger.NopLogger,
 	}
 	addr, shutdown, err := pgtest.ServeTLS(":0", server)
 	if err != nil {
@@ -197,7 +197,7 @@ func TestPSQLQuery(t *testing.T) {
 		}),
 		TypeEngine:     pg.PrimitiveTypeEngine{},
 		StartupTimeout: time.Second,
-		Logger:         logger.NewLogfLogger(t),
+		Logger:         logger.NopLogger,
 	}
 	addr, shutdown, err := pgtest.ServeTCP(":0", server)
 	if err != nil {

rbf/tx_test.go

@@ -94,7 +94,7 @@ func TestTx_CommitRollback(t *testing.T) {
 	})
 
 	t.Run("SingleWriter", func(t *testing.T) {
-		t.Skip("NEED TO FIX IN RACE") //TODO (twg)
+		//t.Skip("NEED TO FIX IN RACE") //TODO (twg)
 		db := MustOpenDB(t)
 		defer MustCloseDB(t, db)
 

rrtx.go

@@ -369,6 +369,7 @@ func (db *RoaringStore) DeleteField(index, field, fieldPath string) error {
 }
 
 // frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
+// The fragment should be closed before this.
 func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
 
 	fragment, ok := frag.(*fragment)
@@ -376,11 +377,6 @@ func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64,
 		return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
 	}
 
-	// Close data files before deletion.
-	if err := fragment.Close(); err != nil {
-		return errors.Wrap(err, "closing fragment")
-	}
-
 	// Delete fragment file.
 	if err := os.Remove(fragment.path); err != nil {
 		return errors.Wrap(err, "deleting fragment file")

server/cluster_test.go

@@ -35,7 +35,7 @@ import (
 // Ensure program can send/receive broadcast messages.
 func TestMain_SendReceiveMessage(t *testing.T) {
 	ms := test.MustRunCluster(t, 2)
-	m0, m1 := ms[0], ms[1]
+	m0, m1 := ms.GetNode(0), ms.GetNode(1)
 	defer ms.Close()
 
 	// Expected indexes and Fields
@@ -131,10 +131,10 @@ func TestClusterResize_EmptyNodes(t *testing.T) {
 	clus := test.MustRunCluster(t, 2)
 	defer clus.Close()
 
-	if clus[0].API.State() != pilosa.ClusterStateNormal {
-		t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State())
-	} else if clus[1].API.State() != pilosa.ClusterStateNormal {
-		t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State())
+	if clus.GetNode(0).API.State() != pilosa.ClusterStateNormal {
+		t.Fatalf("unexpected node0 cluster state: %s", clus.GetNode(0).API.State())
+	} else if clus.GetNode(1).API.State() != pilosa.ClusterStateNormal {
+		t.Fatalf("unexpected node1 cluster state: %s", clus.GetNode(1).API.State())
 	}
 }
 
@@ -144,15 +144,15 @@ func TestClusterResize_AddNode(t *testing.T) {
 		clus := test.MustRunCluster(t, 2)
 		defer clus.Close()
 
-		if !test.CheckClusterState(clus[0], pilosa.ClusterStateNormal, 1000) {
-			t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State())
-		} else if !test.CheckClusterState(clus[1], pilosa.ClusterStateNormal, 1000) {
-			t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State())
+		if !test.CheckClusterState(clus.GetNode(0), pilosa.ClusterStateNormal, 1000) {
+			t.Fatalf("unexpected node0 cluster state: %s", clus.GetNode(0).API.State())
+		} else if !test.CheckClusterState(clus.GetNode(1), pilosa.ClusterStateNormal, 1000) {
+			t.Fatalf("unexpected node1 cluster state: %s", clus.GetNode(1).API.State())
 		}
 	})
 	t.Run("WithIndex", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -168,7 +168,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		}
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -198,7 +198,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		skipTestUnderBlueGreenWithRoaring(t)
 
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -229,7 +229,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -250,7 +250,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 	})
 	t.Run("OneShard", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -278,7 +278,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -299,7 +299,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 	})
 	t.Run("SkippedShard", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -331,7 +331,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -356,7 +356,7 @@ func TestClusterResize_AddNode(t *testing.T) {
 func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 	t.Run("WithIndex", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -378,7 +378,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		}()
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -399,7 +399,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 	})
 	t.Run("ContinuousShards", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -431,7 +431,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		err := m1.Start()
@@ -457,7 +457,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 	})
 	t.Run("SkippedShard", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -489,7 +489,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		errc := make(chan error, 1)
@@ -515,7 +515,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 	})
 	t.Run("WithIndexKeys", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -545,7 +545,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 		m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		m1.Config.Gossip.Port = "0"
 		m1.Config.Gossip.Seeds = []string{seed}
 		errc := make(chan error, 1)
@@ -573,7 +573,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
 func TestCluster_GossipMembership(t *testing.T) {
 	t.Run("Node0Down", func(t *testing.T) {
 		// Configure node0
-		m0 := test.MustRunCluster(t, 1)[0]
+		m0 := test.MustRunCluster(t, 1).GetNode(0)
 		defer m0.Close()
 
 		seed := m0.GossipAddress()
@@ -581,7 +581,7 @@ func TestCluster_GossipMembership(t *testing.T) {
 		var eg errgroup.Group
 
 		// Configure node1
-		m1 := test.NewCommandNode(false)
+		m1 := test.NewCommandNode(t, false)
 		defer m1.Close()
 		eg.Go(func() error {
 			m1.Config.Gossip.Port = "0"
@@ -595,7 +595,7 @@ func TestCluster_GossipMembership(t *testing.T) {
 		})
 
 		// Configure node1
-		m2 := test.NewCommandNode(false)
+		m2 := test.NewCommandNode(t, false)
 		defer m2.Close()
 		eg.Go(func() error {
 			m2.Config.Gossip.Port = "0"
@@ -630,8 +630,8 @@ func TestCluster_GossipMembership(t *testing.T) {
 func TestClusterResize_RemoveNode(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
-	m0 := cluster[0]
-	m1 := cluster[1]
+	m0 := cluster.GetNode(0)
+	m1 := cluster.GetNode(1)
 
 	mustNodeID := func(baseURL string) string {
 		body := test.Do(t, "GET", fmt.Sprintf("%s/status", baseURL), "").Body
@@ -730,7 +730,7 @@ func TestClusterMutualTLS(t *testing.T) {
 
 	cluster := test.MustRunCluster(t, 3, commandOpts...)
 	defer cluster.Close()
-	m0 := cluster[0]
+	m0 := cluster.GetNode(0)
 
 	client0 := m0.Client()
 	if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {

server/grpc.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"net"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pilosa/pilosa/v2"
@@ -909,6 +910,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe
 
 type grpcServer struct {
 	api        *pilosa.API
+	mu         sync.Mutex
 	grpcServer *grpc.Server
 	ln         net.Listener
 
@@ -956,11 +958,13 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error {
 	}
 
 	// create grpc server
+	s.mu.Lock()
 	s.grpcServer = grpc.NewServer(opts...)
 	pb.RegisterPilosaServer(s.grpcServer, NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats))
 
 	// register the server so its services are available to grpc_cli and others
 	reflection.Register(s.grpcServer)
+	s.mu.Unlock()
 
 	// and start...
 	if err := s.grpcServer.Serve(s.ln); err != nil {
@@ -969,6 +973,16 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error {
 	return nil
 }
 
+// Stop stops the GRPC server. There's no error because the underlying GRPC
+// stuff doesn't report an error.
+func (s *grpcServer) Stop() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.grpcServer != nil {
+		s.grpcServer.Stop()
+	}
+}
+
 func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {
 	server := &grpcServer{
 		logger: logger.NopLogger,

server/handler_test.go

@@ -43,7 +43,7 @@ import (
 func TestHandler_PostSchemaCluster(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	h := cmd.Handler.(*http.Handler).Handler
 
 	t.Run("PostSchema", func(t *testing.T) {
@@ -56,8 +56,8 @@ func TestHandler_PostSchemaCluster(t *testing.T) {
 			}
 			t.Fatalf("unexpected code: %v, bod: %s", w.Code, bod)
 		}
-		for i := 0; i < len(cluster); i++ {
-			cmd = cluster[i]
+		for i := 0; i < cluster.Len(); i++ {
+			cmd = cluster.GetNode(i)
 			idx, err := cmd.API.Index(context.Background(), "blah")
 			if err != nil {
 				t.Fatalf("getting index: %v", err)
@@ -82,7 +82,7 @@ func TestHandler_PostSchemaCluster(t *testing.T) {
 func TestHandler_Endpoints(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	h := cmd.Handler.(*http.Handler).Handler
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}
@@ -1049,7 +1049,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})})
 		defer clus.Close()
 		w = httptest.NewRecorder()
-		h := clus[0].Handler.(*http.Handler).Handler
+		h := clus.GetNode(0).Handler.(*http.Handler).Handler
 		h.ServeHTTP(w, req)
 		result = w.Result()
 
@@ -1297,59 +1297,59 @@ func TestHandler_Endpoints(t *testing.T) {
 }
 
 func TestCluster_TranslateStore(t *testing.T) {
-	cluster := make(test.Cluster, 1)
-	cluster[0] = test.NewCommandNode(true,
+	cluster := test.MustNewCluster(t, 1)
+	cluster.Nodes[0] = test.NewCommandNode(t, true,
 		server.OptCommandServerOptions(
 			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
 			pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})),
 		),
 	)
-	cluster[0].Config.Gossip.Port = "0"
-	err := cluster[0].Start()
+	cluster.GetNode(0).Config.Gossip.Port = "0"
+	err := cluster.GetNode(0).Start()
 	if err != nil {
 		t.Fatalf("starting cluster 0: %v", err)
 	}
-	defer cluster[0].Close()
+	defer cluster.GetNode(0).Close()
 
-	test.Do(t, "POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
 }
 
 func TestClusterTranslator(t *testing.T) {
-	cluster := make(test.Cluster, 2)
-	cluster[0] = test.NewCommandNode(true,
+	cluster := test.MustNewCluster(t, 2)
+	cluster.Nodes[0] = test.NewCommandNode(t, true,
 		server.OptCommandServerOptions(
 			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
 		),
 	)
-	cluster[0].Config.Gossip.Port = "0"
-	err := cluster[0].Start()
+	cluster.GetNode(0).Config.Gossip.Port = "0"
+	err := cluster.GetNode(0).Start()
 	if err != nil {
 		t.Fatalf("starting cluster 0: %v", err)
 	}
-	defer cluster[0].Close()
-	cluster[1] = test.NewCommandNode(false,
+	defer cluster.GetNode(0).Close()
+	cluster.Nodes[1] = test.NewCommandNode(t, false,
 		server.OptCommandServerOptions(
 			pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
 			pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})),
 		),
 	)
-	cluster[1].Config.Gossip.Port = "0"
-	cluster[1].Config.Gossip.Seeds = []string{cluster[0].GossipAddress()}
-	err = cluster[1].Start()
+	cluster.GetNode(1).Config.Gossip.Port = "0"
+	cluster.GetNode(1).Config.Gossip.Seeds = []string{cluster.GetNode(0).GossipAddress()}
+	err = cluster.GetNode(1).Start()
 	if err != nil {
 		t.Fatalf("starting cluster 1: %v", err)
 	}
-	defer cluster[1].Close()
+	defer cluster.GetNode(1).Close()
 
-	test.Do(t, "POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
-	test.Do(t, "POST", cluster[0].URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}")
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}")
 
-	test.Do(t, "POST", cluster[0].URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")")
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")")
 
 	var result0, result1 string
 	if err := test.RetryUntil(2*time.Second, func() error {
-		result0 = test.Do(t, "POST", cluster[0].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
-		result1 = test.Do(t, "POST", cluster[1].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
+		result0 = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
+		result1 = test.Do(t, "POST", cluster.GetNode(1).URL()+"/index/i0/query", "Row(f0=\"bar\")").Body
 		if result0 != result1 {
 			return fmt.Errorf("`%s` != `%s`", result0, result1)
 		}

server/server.go

@@ -49,6 +49,7 @@ import (
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pilosa/pilosa/v2/statsd"
 	"github.com/pilosa/pilosa/v2/syswrap"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -196,6 +197,7 @@ func (m *Command) Start() (err error) {
 		}
 	}
 
+	_ = testhook.Opened(pilosa.NewAuditor(), m, nil)
 	close(m.Started)
 	return nil
 }
@@ -537,6 +539,7 @@ func (m *Command) GossipTransport() *gossip.Transport {
 func (m *Command) Close() error {
 	defer close(m.done)
 	eg := errgroup.Group{}
+	m.grpcServer.Stop()
 	eg.Go(m.Handler.Close)
 	eg.Go(m.Server.Close)
 	eg.Go(m.API.Close)
@@ -552,6 +555,7 @@ func (m *Command) Close() error {
 	}
 
 	err := eg.Wait()
+	_ = testhook.Closed(pilosa.NewAuditor(), m, nil)
 	return errors.Wrap(err, "closing everything")
 }
 

server/server_test.go

@@ -355,7 +355,7 @@ func TestConcurrentFieldCreation(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
 
-	api0 := cluster[0].API
+	api0 := cluster.GetNode(0).API
 	if _, err := api0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
@@ -379,10 +379,10 @@ func TestTransactionsAPI(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
 
-	api0 := cluster[0].API
-	api1 := cluster[1].API
+	api0 := cluster.GetNode(0).API
+	api1 := cluster.GetNode(1).API
 	ctx := context.Background()
-	//api2 := cluster[2].API
+	//api2 := cluster.GetNode(2).API
 
 	// can fetch empty transactions
 	if trnsMap, err := api0.Transactions(ctx); err != nil {
@@ -508,7 +508,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
 	defer cluster.Close()
 
 	// Create the schema.
-	client0 := cluster[0].Client()
+	client0 := cluster.GetNode(0).Client()
 	if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
 		t.Fatal("create index:", err)
 	}
@@ -523,12 +523,12 @@ func TestMain_RecalculateHashes(t *testing.T) {
 			data = append(data, fmt.Sprintf(`Set(%d, f=%d)`, columnID, rowID))
 		}
 	}
-	if _, err := cluster[0].Query(t, "i", "", strings.Join(data, "")); err != nil {
+	if _, err := cluster.GetNode(0).Query(t, "i", "", strings.Join(data, "")); err != nil {
 		t.Fatal("setting columns:", err)
 	}
 
 	// Calculate caches on the first node
-	err := cluster[0].RecalculateCaches(t)
+	err := cluster.GetNode(0).RecalculateCaches(t)
 	if err != nil {
 		t.Fatalf("recalculating caches: %v", err)
 	}
@@ -536,7 +536,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
 	target := `{"results":[[{"id":7,"key":"","count":99},{"id":1,"key":"","count":99},{"id":9,"key":"","count":99},{"id":5,"key":"","count":99},{"id":4,"key":"","count":99},{"id":8,"key":"","count":99},{"id":2,"key":"","count":99},{"id":6,"key":"","count":99},{"id":3,"key":"","count":99}]]}`
 
 	// Run a TopN query on all nodes. The result should be the same as the target.
-	for _, m := range cluster {
+	for _, m := range cluster.Nodes {
 		res, err := m.Query(t, "i", "", `TopN(f)`)
 		if err != nil {
 			t.Fatal(err)
@@ -635,27 +635,27 @@ func TestClusteringNodesReplica1(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	if err := cluster[2].Command.Close(); err != nil {
+	if err := cluster.GetNode(2).Command.Close(); err != nil {
 		t.Fatalf("closing third node: %v", err)
 	}
 
 	// confirm that cluster stops accepting queries after one node closes
-	if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
+	if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
 		t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
 	}
 
 	// Create new main with the same config.
-	config := cluster[2].Command.Config
+	config := cluster.GetNode(2).Command.Config
 	config.Translation.MapSize = 100000
 
 	// this isn't necessary, but makes the test run way faster
-	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
+	config.Gossip.Port = strconv.Itoa(int(cluster.GetNode(2).Command.GossipTransport().URI.Port))
 
-	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
-	cluster[2].Command.Config = config
+	cluster.GetNode(2).Command = server.NewCommand(cluster.GetNode(2).Stdin, cluster.GetNode(2).Stdout, cluster.GetNode(2).Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
+	cluster.GetNode(2).Command.Config = config
 
 	// Run new program.
-	if err := cluster[2].Start(); err != nil {
+	if err := cluster.GetNode(2).Start(); err != nil {
 		t.Fatalf("restarting node 2: %v", err)
 	}
 
@@ -667,7 +667,7 @@ func TestClusteringNodesReplica1(t *testing.T) {
 
 func TestClusteringNodesReplica2(t *testing.T) {
 	cluster := test.MustNewCluster(t, 3)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -681,7 +681,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	if err := cluster[2].Command.Close(); err != nil {
+	if err := cluster.GetNode(2).Command.Close(); err != nil {
 		t.Fatalf("closing third node: %v", err)
 	}
 
@@ -691,12 +691,12 @@ func TestClusteringNodesReplica2(t *testing.T) {
 	}
 
 	// confirm that cluster keeps accepting queries if replication > 1
-	if _, err := cluster[0].API.CreateIndex(context.Background(), "anewindex", pilosa.IndexOptions{}); err != nil {
+	if _, err := cluster.GetNode(0).API.CreateIndex(context.Background(), "anewindex", pilosa.IndexOptions{}); err != nil {
 		t.Fatalf("got unexpected error creating index: %v", err)
 	}
 
 	// confirm that cluster stops accepting queries if 2 nodes fail and replication == 2
-	if err := cluster[1].Command.Close(); err != nil {
+	if err := cluster.GetNode(1).Command.Close(); err != nil {
 		t.Fatalf("closing 2nd node: %v", err)
 	}
 
@@ -705,23 +705,23 @@ func TestClusteringNodesReplica2(t *testing.T) {
 		t.Fatalf("after closing second server: %v", err)
 	}
 
-	if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
+	if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
 		t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
 	}
 
 	// Create new main with the same config.
-	config := cluster[2].Command.Config
+	config := cluster.GetNode(2).Command.Config
 	config.Translation.MapSize = 100000
-	// config.Bind = cluster[2].API.Node().URI.HostPort()
+	// config.Bind = cluster.GetNode(2).API.Node().URI.HostPort()
 
 	// this isn't necessary, but makes the test run way faster
-	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
+	config.Gossip.Port = strconv.Itoa(int(cluster.GetNode(2).Command.GossipTransport().URI.Port))
 
-	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
-	cluster[2].Command.Config = config
+	cluster.GetNode(2).Command = server.NewCommand(cluster.GetNode(2).Stdin, cluster.GetNode(2).Stdout, cluster.GetNode(2).Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
+	cluster.GetNode(2).Command.Config = config
 
 	// Run new program.
-	if err := cluster[2].Start(); err != nil {
+	if err := cluster.GetNode(2).Start(); err != nil {
 		t.Fatalf("restarting node 2: %v", err)
 	}
 
@@ -731,18 +731,18 @@ func TestClusteringNodesReplica2(t *testing.T) {
 	}
 
 	// Create new main with the same config.
-	config = cluster[1].Command.Config
-	// config.Bind = cluster[1].API.Node().URI.HostPort()
+	config = cluster.GetNode(1).Command.Config
+	// config.Bind = cluster.GetNode(1).API.Node().URI.HostPort()
 	config.Translation.MapSize = 100000
 
 	// this isn't necessary, but makes the test run way faster
-	config.Gossip.Port = strconv.Itoa(int(cluster[1].Command.GossipTransport().URI.Port))
+	config.Gossip.Port = strconv.Itoa(int(cluster.GetNode(1).Command.GossipTransport().URI.Port))
 
-	cluster[1].Command = server.NewCommand(cluster[1].Stdin, cluster[1].Stdout, cluster[1].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
-	cluster[1].Command.Config = config
+	cluster.GetNode(1).Command = server.NewCommand(cluster.GetNode(1).Stdin, cluster.GetNode(1).Stdout, cluster.GetNode(1).Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
+	cluster.GetNode(1).Command.Config = config
 
 	// Run new program.
-	if err := cluster[1].Start(); err != nil {
+	if err := cluster.GetNode(1).Start(); err != nil {
 		t.Fatalf("restarting node 1: %v", err)
 	}
 
@@ -754,7 +754,7 @@ func TestClusteringNodesReplica2(t *testing.T) {
 
 func TestRemoveNodeAfterItDies(t *testing.T) {
 	cluster := test.MustNewCluster(t, 3)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -774,10 +774,9 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	// prevent double-closing cluster[2] from the deferred Close above
-	disabled, cluster := cluster[2], cluster[:2]
-
-	if err := disabled.Command.Close(); err != nil {
+	// prevent double-closing cluster.GetNode(2) from the deferred Close above
+	disabled := cluster.GetNode(2)
+	if err := cluster.CloseAndRemove(2); err != nil {
 		t.Fatalf("closing third node: %v", err)
 	}
 
@@ -786,7 +785,7 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	if _, err := cluster[0].API.RemoveNode(disabled.API.Node().ID); err != nil {
+	if _, err := cluster.GetNode(0).API.RemoveNode(disabled.API.Node().ID); err != nil {
 		t.Fatalf("removing failed node: %v", err)
 	}
 
@@ -795,7 +794,7 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
 		t.Fatalf("removing disabled node: %v", err)
 	}
 
-	hosts := cluster[0].API.Hosts(context.Background())
+	hosts := cluster.GetNode(0).API.Hosts(context.Background())
 	if len(hosts) != 2 {
 		t.Fatalf("unexpected hosts: %v", hosts)
 	}
@@ -803,7 +802,7 @@ func TestRemoveNodeAfterItDies(t *testing.T) {
 
 func TestRemoveConcurrentIndexCreation(t *testing.T) {
 	cluster := test.MustNewCluster(t, 3)
-	for _, c := range cluster {
+	for _, c := range cluster.Nodes {
 		c.Config.Cluster.ReplicaN = 2
 	}
 	err := cluster.Start()
@@ -818,11 +817,11 @@ func TestRemoveConcurrentIndexCreation(t *testing.T) {
 
 	errc := make(chan error)
 	go func() {
-		_, err := cluster[0].API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{})
+		_, err := cluster.GetNode(0).API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{})
 		errc <- err
 	}()
 
-	if _, err := cluster[0].API.RemoveNode(cluster[2].API.Node().ID); err != nil {
+	if _, err := cluster.GetNode(0).API.RemoveNode(cluster.GetNode(2).API.Node().ID); err != nil {
 		t.Fatalf("removing node: %v", err)
 	}
 
@@ -831,7 +830,7 @@ func TestRemoveConcurrentIndexCreation(t *testing.T) {
 		t.Fatalf("starting cluster: %v", err)
 	}
 
-	hosts := cluster[0].API.Hosts(context.Background())
+	hosts := cluster.GetNode(0).API.Hosts(context.Background())
 	if len(hosts) != 2 {
 		t.Fatalf("unexpected hosts: %v", hosts)
 	}
@@ -948,9 +947,9 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) {
 func TestClusterQueriesAfterRestart(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
-	cmd1 := cluster[1]
+	cmd1 := cluster.GetNode(1)
 
-	for _, com := range cluster {
+	for _, com := range cluster.Nodes {
 		nodes := com.API.Hosts(context.Background())
 		for _, n := range nodes {
 			if n.State != "READY" {
@@ -992,7 +991,7 @@ func TestClusterQueriesAfterRestart(t *testing.T) {
 	}
 
 	// confirm that cluster stops accepting queries after one node closes
-	if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
+	if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") {
 		t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
 	}
 
@@ -1034,9 +1033,9 @@ func TestClusterExhaustingConnections(t *testing.T) {
 	}
 	cluster := test.MustRunCluster(t, 5)
 	defer cluster.Close()
-	cmd1 := cluster[1]
+	cmd1 := cluster.GetNode(1)
 
-	for _, com := range cluster {
+	for _, com := range cluster.Nodes {
 		nodes := com.API.Hosts(context.Background())
 		for _, n := range nodes {
 			if n.State != "READY" {
@@ -1053,7 +1052,7 @@ func TestClusterExhaustingConnections(t *testing.T) {
 		i := i
 		eg.Go(func() error {
 			for j := i; j < 10000; j += 20 {
-				_, err := cluster[i%5].API.Query(context.Background(), &pilosa.QueryRequest{
+				_, err := cluster.GetNode(i%5).API.Query(context.Background(), &pilosa.QueryRequest{
 					Index: "testidx",
 					Query: fmt.Sprintf("Set(%d, testfield=0)", j*pilosa.ShardWidth),
 				})
@@ -1120,9 +1119,9 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) {
 	}
 	cluster := test.MustRunCluster(t, 5)
 	defer cluster.Close()
-	cmd1 := cluster[1]
+	cmd1 := cluster.GetNode(1)
 
-	for _, com := range cluster {
+	for _, com := range cluster.Nodes {
 		nodes := com.API.Hosts(context.Background())
 		for _, n := range nodes {
 			if n.State != "READY" {
@@ -1151,7 +1150,7 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) {
 				if (j-i)%1000 == 0 {
 					fmt.Printf("%d is %.2f%% done.\n", i, float64(j-i)*100/100000)
 				}
-				err := cluster[i%5].API.ImportRoaring(context.Background(), "testidx", "testfield", j, false, &pilosa.ImportRoaringRequest{
+				err := cluster.GetNode(int(i%5)).API.ImportRoaring(context.Background(), "testidx", "testfield", j, false, &pilosa.ImportRoaringRequest{
 					Views: map[string][]byte{
 						"": data,
 					},
@@ -1172,12 +1171,12 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) {
 func TestClusterMinMaxSumDecimal(t *testing.T) {
 	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 
 	cmd.MustCreateIndex(t, "testdec", pilosa.IndexOptions{Keys: true, TrackExistence: true})
 	cmd.MustCreateField(t, "testdec", "adec", pilosa.OptFieldTypeDecimal(2))
 
-	test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", `
+	test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", `
 Set("a", adec=42.2)
 Set("b", adec=11.12)
 Set("c", adec=13.41)
@@ -1188,21 +1187,21 @@ Set("g", adec=15.52)
 Set("h", adec=100.22)
 `)
 
-	result := test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", "Sum(field=adec)")
+	result := test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Sum(field=adec)")
 	if !strings.Contains(result.Body, `"decimalValue":305.59`) {
 		t.Fatalf("expected decimal sum of 305.59, but got: '%s'", result.Body)
 	} else if !strings.Contains(result.Body, `"count":8`) {
 		t.Fatalf("expected count 8, but got: '%s'", result.Body)
 	}
 
-	result = test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", "Max(field=adec)")
+	result = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Max(field=adec)")
 	if !strings.Contains(result.Body, `"decimalValue":100.22`) {
 		t.Fatalf("expected decimal max of 100.22, but got: '%s'", result.Body)
 	} else if !strings.Contains(result.Body, `"count":1`) {
 		t.Fatalf("expected count 1, but got: '%s'", result.Body)
 	}
 
-	result = test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", "Min(field=adec)")
+	result = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Min(field=adec)")
 	if !strings.Contains(result.Body, `"decimalValue":11.12`) {
 		t.Fatalf("expected decimal min of 11.12, but got: '%s'", result.Body)
 	} else if !strings.Contains(result.Body, `"count":1`) {

server_internal_test.go

@@ -15,10 +15,11 @@
 package pilosa
 
 import (
-	"io/ioutil"
 	"runtime"
 	"testing"
 	"time"
+
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Ensure the file handle count is working
@@ -40,7 +41,7 @@ func TestCountOpenFiles(t *testing.T) {
 
 func TestMonitorAntiEntropyZero(t *testing.T) {
 
-	td, err := ioutil.TempDir(*TempDir, "")
+	td, err := testhook.TempDirInDir(t, *TempDir, "")
 	if err != nil {
 		t.Fatalf("getting temp dir: %v", err)
 	}
@@ -49,6 +50,7 @@ func TestMonitorAntiEntropyZero(t *testing.T) {
 	if err != nil {
 		t.Fatalf("making new server: %v", err)
 	}
+	defer s.Close()
 
 	ch := make(chan struct{})
 	go func() {

snapshotqueue.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/pilosa/pilosa/v2/logger"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
@@ -90,7 +91,7 @@ var defaultSnapshotQueue = &queuelessSnapshotQueue{}
 // w worker threads.
 func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue {
 	ctx, cancel := context.WithCancel(context.Background())
-	sq := prioritySnapshotQueue{
+	sq := &prioritySnapshotQueue{
 		normal:     make(chan snapshotRequest, n),
 		urgent:     make(chan snapshotRequest),
 		background: make(chan snapshotRequest),
@@ -102,8 +103,9 @@ func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue {
 	if sq.logger == nil {
 		sq.logger = logger.NewStandardLogger(os.Stderr)
 	}
+	_ = testhook.Opened(NewAuditor(), sq, nil)
 	sq.spawnWorkers(w)
-	return &sq
+	return sq
 }
 
 type snapshotRequest struct {
@@ -136,6 +138,7 @@ type prioritySnapshotQueue struct {
 		enqueued uint32
 		skipped  uint32
 	}
+	stopped bool
 }
 
 func (sq *prioritySnapshotQueue) spawnWorkers(w int) {
@@ -205,6 +208,10 @@ func (sq *prioritySnapshotQueue) process(req snapshotRequest) {
 func (sq *prioritySnapshotQueue) Stop() {
 	sq.mu.Lock()
 	defer sq.mu.Unlock()
+	if sq.stopped {
+		return
+	}
+	sq.stopped = true
 	sq.cancel()
 	// scanners need to be done before we close the other channels.
 	sq.scanWG.Wait()
@@ -214,6 +221,7 @@ func (sq *prioritySnapshotQueue) Stop() {
 	sq.urgent = nil
 	close(sq.background)
 	sq.background = nil
+	_ = testhook.Closed(NewAuditor(), sq, nil)
 	enqueued := atomic.LoadUint32(&sq.stats.enqueued)
 	skipped := atomic.LoadUint32(&sq.stats.skipped)
 	if skipped > 0 || enqueued > 1 {

stats/stats_test.go

@@ -32,7 +32,7 @@ import (
 // TestMultiStatClient_Expvar run the multistat client with exp var
 // since the EXPVAR data is stored in a global we should run these in one test function
 func TestMultiStatClient_Expvar(t *testing.T) {
-	hldr := test.MustOpenHolder()
+	hldr := test.MustOpenHolder(t)
 	defer hldr.Close()
 
 	c := stats.NewExpvarStatsClient()
@@ -93,7 +93,7 @@ func TestMultiStatClient_Expvar(t *testing.T) {
 func TestStatsCount_TopN(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-	hldr := test.Holder{Holder: c[0].Server.Holder()}
+	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
 
 	hldr.SetBit("d", "f", 0, 0)
 	hldr.SetBit("d", "f", 0, 1)
@@ -115,7 +115,7 @@ func TestStatsCount_TopN(t *testing.T) {
 			called = true
 		},
 	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil {
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil {
 		t.Fatal(err)
 	}
 	if !called {
@@ -126,7 +126,7 @@ func TestStatsCount_TopN(t *testing.T) {
 func TestStatsCount_Bitmap(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-	hldr := test.Holder{Holder: c[0].Server.Holder()}
+	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
 
 	hldr.SetBit("d", "f", 0, 0)
 	hldr.SetBit("d", "f", 0, 1)
@@ -144,7 +144,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
 			called = true
 		},
 	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil {
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil {
 		t.Fatal(err)
 	}
 	if !called {
@@ -155,7 +155,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
 func TestStatsCount_SetRowAttrsBulk(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-	hldr := test.Holder{Holder: c[0].Server.Holder()}
+	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
 
 	hldr.SetBit("d", "f", 10, 0)
 	hldr.SetBit("d", "f", 10, 1)
@@ -178,7 +178,7 @@ func TestStatsCount_SetRowAttrsBulk(t *testing.T) {
 			called = true
 		},
 	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil {
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil {
 		t.Fatal(err)
 	}
 	if !called {
@@ -189,7 +189,7 @@ func TestStatsCount_SetRowAttrsBulk(t *testing.T) {
 func TestStatsCount_SetColumnAttrs(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
-	hldr := test.Holder{Holder: c[0].Server.Holder()}
+	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
 
 	hldr.SetBit("d", "f", 10, 0)
 	hldr.SetBit("d", "f", 10, 1)
@@ -212,7 +212,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
 			called = true
 		},
 	}
-	if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetColumnAttrs(10, foo="bar")`}); err != nil {
+	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetColumnAttrs(10, foo="bar")`}); err != nil {
 		t.Fatal(err)
 	}
 	if !called {
@@ -223,7 +223,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) {
 func TestStatsCount_APICalls(t *testing.T) {
 	cluster := test.MustRunCluster(t, 1)
 	defer cluster.Close()
-	cmd := cluster[0]
+	cmd := cluster.GetNode(0)
 	h := cmd.Handler.(*http.Handler).Handler
 	holder := cmd.Server.Holder()
 	hldr := test.Holder{Holder: holder}

test/cluster.go

@@ -35,20 +35,34 @@ type ModHasher struct{}
 func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
 
 // Cluster represents a Pilosa cluster (multiple Command instances)
-type Cluster []*Command
+type Cluster struct {
+	Nodes []*Command
+}
 
 // Query executes an API.Query through one of the cluster's node's API. It fails
 // the test if there is an error.
-func (c Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse {
+func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse {
 	t.Helper()
-	if len(c) == 0 {
+	if len(c.Nodes) == 0 {
 		t.Fatal("must have at least one node in cluster to query")
 	}
 
-	return c[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query})
+	return c.Nodes[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query})
 }
 
-func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) {
+func (c *Cluster) GetNode(n int) *Command {
+	return c.Nodes[n]
+}
+
+func (c *Cluster) GetHolder(n int) *Holder {
+	return &Holder{Holder: c.Nodes[n].Server.Holder()}
+}
+
+func (c *Cluster) Len() int {
+	return len(c.Nodes)
+}
+
+func (c *Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) {
 	t.Helper()
 	byShard := make(map[uint64][][2]uint64)
 	for _, rowcol := range rowcols {
@@ -63,7 +77,7 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint
 			rowIDs[i] = bit[0]
 			colIDs[i] = bit[1]
 		}
-		nodes, err := c[0].API.ShardNodes(context.Background(), index, shard)
+		nodes, err := c.Nodes[0].API.ShardNodes(context.Background(), index, shard)
 		if err != nil {
 			t.Fatalf("getting shard nodes: %v", err)
 		}
@@ -72,7 +86,7 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint
 		// suggesting that elsewhere we would support importing to a
 		// single node, regardless of where the data ends up.
 		for _, node := range nodes {
-			for _, com := range c {
+			for _, com := range c.Nodes {
 				if com.API.Node().ID != node.ID {
 					continue
 				}
@@ -92,13 +106,13 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint
 }
 
 // CreateField creates the index (if necessary) and field specified.
-func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field {
+func (c *Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field {
 	t.Helper()
-	idx, err := c[0].API.CreateIndex(context.Background(), index, iopts)
+	idx, err := c.Nodes[0].API.CreateIndex(context.Background(), index, iopts)
 	if err != nil && !strings.Contains(err.Error(), "index already exists") {
 		t.Fatalf("creating index: %v", err)
 	} else if err != nil { // index exists
-		idx, err = c[0].API.Index(context.Background(), index)
+		idx, err = c.Nodes[0].API.Index(context.Background(), index)
 		if err != nil {
 			t.Fatalf("getting index: %v", err)
 		}
@@ -107,7 +121,7 @@ func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptio
 		t.Logf("existing index options:\n%v\ndon't match given opts:\n%v\n in pilosa/test.Cluster.CreateField", idx.Options(), iopts)
 	}
 
-	f, err := c[0].API.CreateField(context.Background(), index, field, fopts...)
+	f, err := c.Nodes[0].API.CreateField(context.Background(), index, field, fopts...)
 	// we'll assume the field doesn't exist because checking if the options
 	// match seems painful.
 	if err != nil {
@@ -117,9 +131,9 @@ func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptio
 }
 
 // Start runs a Cluster
-func (c Cluster) Start() error {
-	var gossipSeeds = make([]string, len(c))
-	for i, cc := range c {
+func (c *Cluster) Start() error {
+	var gossipSeeds = make([]string, len(c.Nodes))
+	for i, cc := range c.Nodes {
 		cc.Config.Gossip.Port = "0"
 		cc.Config.Gossip.Seeds = gossipSeeds[:i]
 		if err := cc.Start(); err != nil {
@@ -131,8 +145,8 @@ func (c Cluster) Start() error {
 }
 
 // Stop stops a Cluster
-func (c Cluster) Close() error {
-	for i, cc := range c {
+func (c *Cluster) Close() error {
+	for i, cc := range c.Nodes {
 		if err := cc.Close(); err != nil {
 			return errors.Wrapf(err, "stopping server %d", i)
 		}
@@ -140,19 +154,30 @@ func (c Cluster) Close() error {
 	return nil
 }
 
+func (c *Cluster) CloseAndRemove(n int) error {
+	if n < 0 || n >= len(c.Nodes) {
+		return fmt.Errorf("close/remove from cluster: index %d out of range (len %d)", n, len(c.Nodes))
+	}
+	err := c.Nodes[n].Close()
+	copy(c.Nodes[n:], c.Nodes[n+1:])
+	c.Nodes = c.Nodes[:len(c.Nodes)-1]
+	return err
+}
+
 // AwaitState waits for the cluster coordinator (assumed to be the first
 // node) to reach a specified state.
-func (c Cluster) AwaitCoordinatorState(expectedState string, timeout time.Duration) error {
-	if len(c) < 1 {
+func (c *Cluster) AwaitCoordinatorState(expectedState string, timeout time.Duration) error {
+	if len(c.Nodes) < 1 {
 		return errors.New("can't await coordinator state on an empty cluster")
 	}
-	return c[:1].AwaitState(expectedState, timeout)
+	onlyCoordinator := &Cluster{Nodes: c.Nodes[:1]}
+	return onlyCoordinator.AwaitState(expectedState, timeout)
 }
 
 // ExceptionalState returns an error if any node in the cluster is not
 // in the expected state.
-func (c Cluster) ExceptionalState(expectedState string) error {
-	for _, node := range c {
+func (c *Cluster) ExceptionalState(expectedState string) error {
+	for _, node := range c.Nodes {
 		state := node.API.State()
 		if state != expectedState {
 			return fmt.Errorf("node %q: state %s", node.ID(), state)
@@ -162,8 +187,8 @@ func (c Cluster) ExceptionalState(expectedState string) error {
 }
 
 // AwaitState waits for the whole cluster to reach a specified state.
-func (c Cluster) AwaitState(expectedState string, timeout time.Duration) (err error) {
-	if len(c) < 1 {
+func (c *Cluster) AwaitState(expectedState string, timeout time.Duration) (err error) {
+	if len(c.Nodes) < 1 {
 		return errors.New("can't await state of an empty cluster")
 	}
 	startTime := time.Now()
@@ -184,7 +209,7 @@ func (c Cluster) AwaitState(expectedState string, timeout time.Duration) (err er
 // slice of command options, those options are used with every node.
 // If it is empty, default options are used. Otherwise, it must contain size
 // slices of command options, which are used with corresponding nodes.
-func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
+func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cluster {
 	tb.Helper()
 	c, err := newCluster(tb, size, opts...)
 	if err != nil {
@@ -206,7 +231,7 @@ func CheckClusterState(m *Command, state string, n int) bool {
 }
 
 // newCluster creates a new cluster
-func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluster, error) {
+func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Cluster, error) {
 	if size == 0 {
 		return nil, errors.New("cluster must contain at least one node")
 	}
@@ -214,26 +239,26 @@ func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluste
 		return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
 	}
 
-	cluster := make(Cluster, size)
+	cluster := &Cluster{Nodes: make([]*Command, size)}
 	name := tb.Name()
 	for i := 0; i < size; i++ {
 		var commandOpts []server.CommandOption
 		if len(opts) > 0 {
 			commandOpts = opts[i%len(opts)]
 		}
-		m := NewCommandNode(i == 0, commandOpts...)
+		m := NewCommandNode(tb, i == 0, commandOpts...)
 		err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name+"_"+strconv.Itoa(i)), 0600)
 		if err != nil {
 			return nil, errors.Wrap(err, "writing node id")
 		}
-		cluster[i] = m
+		cluster.Nodes[i] = m
 	}
 
 	return cluster, nil
 }
 
 // runCluster creates and starts a new cluster
-func runCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluster, error) {
+func runCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Cluster, error) {
 	cluster, err := newCluster(tb, size, opts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "new cluster")
@@ -247,7 +272,7 @@ func runCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluste
 
 // MustRunCluster creates and starts a new cluster. The opts parameter
 // is slightly magical; see MustNewCluster.
-func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
+func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cluster {
 	// We want tests to default to using the in-memory translate store, so we
 	// prepend opts with that functional option. If a different translate store
 	// has been specified, it will override this one.

test/field.go

@@ -15,11 +15,11 @@
 package test
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Field represents a test wrapper for pilosa.Field.
@@ -28,8 +28,8 @@ type Field struct {
 }
 
 // newField returns a new instance of Field d/0.
-func newField(opts pilosa.FieldOption) *Field {
-	path, err := ioutil.TempDir("", "pilosa-field-")
+func newField(tb testing.TB, opts pilosa.FieldOption) *Field {
+	path, err := testhook.TempDir(tb, "pilosa-field-")
 	if err != nil {
 		panic(err)
 	}
@@ -41,8 +41,8 @@ func newField(opts pilosa.FieldOption) *Field {
 }
 
 // mustOpenField returns a new, opened field at a temporary path. Panic on error.
-func mustOpenField(opts pilosa.FieldOption) *Field {
-	f := newField(opts)
+func mustOpenField(tb testing.TB, opts pilosa.FieldOption) *Field {
+	f := newField(tb, opts)
 	if err := f.Open(); err != nil {
 		panic(err)
 	}
@@ -76,7 +76,7 @@ func (f *Field) reopen() error {
 
 // Ensure field can set its cache
 func TestField_SetCacheSize(t *testing.T) {
-	f := mustOpenField(pilosa.OptFieldTypeDefault())
+	f := mustOpenField(t, pilosa.OptFieldTypeDefault())
 	defer f.close()
 	cacheSize := uint32(100)
 

test/holder.go

@@ -15,14 +15,14 @@
 package test
 
 import (
-	"io/ioutil"
 	"math"
-	"os"
+	"testing"
 	"time"
 
 	"github.com/pilosa/pilosa/v2"
 	"github.com/pilosa/pilosa/v2/boltdb"
 	"github.com/pilosa/pilosa/v2/pql"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 var panicOn = pilosa.PanicOn
@@ -33,8 +33,8 @@ type Holder struct {
 }
 
 // NewHolder returns a new instance of Holder with a temporary path.
-func NewHolder() *Holder {
-	path, err := ioutil.TempDir("", "pilosa-")
+func NewHolder(tb testing.TB) *Holder {
+	path, err := testhook.TempDir(tb, "pilosa-holder-")
 	if err != nil {
 		panic(err)
 	}
@@ -47,17 +47,16 @@ func NewHolder() *Holder {
 }
 
 // MustOpenHolder creates and opens a holder at a temporary path. Panic on error.
-func MustOpenHolder() *Holder {
-	h := NewHolder()
+func MustOpenHolder(tb testing.TB) *Holder {
+	h := NewHolder(tb)
 	if err := h.Open(); err != nil {
 		panic(err)
 	}
 	return h
 }
 
-// Close closes the holder and removes all underlying data.
+// Close closes the holder. The data should be removed by the
 func (h *Holder) Close() error {
-	defer os.RemoveAll(h.Path)
 	return h.Holder.Close()
 }
 

test/index.go

@@ -15,10 +15,10 @@
 package test
 
 import (
-	"io/ioutil"
-	"os"
+	"testing"
 
 	"github.com/pilosa/pilosa/v2"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 // Index represents a test wrapper for pilosa.Index.
@@ -27,12 +27,15 @@ type Index struct {
 }
 
 // newIndex returns a new instance of Index.
-func newIndex() *Index {
-	path, err := ioutil.TempDir("", "pilosa-index-")
+func newIndex(tb testing.TB) *Index {
+	path, err := testhook.TempDir(tb, "pilosa-index-")
 	if err != nil {
 		panic(err)
 	}
 	h := pilosa.NewHolder(pilosa.DefaultPartitionN)
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	h.Path = path
 	index, err := h.CreateIndex("i", pilosa.IndexOptions{})
 	if err != nil {
@@ -42,17 +45,13 @@ func newIndex() *Index {
 }
 
 // MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
-func MustOpenIndex() *Index {
-	index := newIndex()
-	if err := index.Open(false); err != nil {
-		panic(err)
-	}
+func MustOpenIndex(tb testing.TB) *Index {
+	index := newIndex(tb)
 	return index
 }
 
 // Close closes the index and removes the underlying data.
 func (i *Index) Close() error {
-	defer os.RemoveAll(i.Path())
 	return i.Index.Close()
 }
 
@@ -70,10 +69,6 @@ func (i *Index) Reopen() error {
 	if err != nil {
 		return err
 	}
-
-	if err := i.Open(false); err != nil {
-		return err
-	}
 	return nil
 }
 

test/pilosa.go

@@ -30,6 +30,7 @@ import (
 	"github.com/pilosa/pilosa/v2/encoding/proto"
 	"github.com/pilosa/pilosa/v2/http"
 	"github.com/pilosa/pilosa/v2/server"
+	"github.com/pilosa/pilosa/v2/testhook"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////
@@ -48,8 +49,8 @@ func OptAllowedOrigins(origins []string) server.CommandOption {
 }
 
 // newCommand returns a new instance of Main with a temporary data directory and random port.
-func newCommand(opts ...server.CommandOption) *Command {
-	path, err := ioutil.TempDir("", "pilosa-")
+func newCommand(tb testing.TB, opts ...server.CommandOption) *Command {
+	path, err := testhook.TempDir(tb, "pilosa-command-")
 	if err != nil {
 		panic(err)
 	}
@@ -85,12 +86,12 @@ func newCommand(opts ...server.CommandOption) *Command {
 }
 
 // NewCommandNode returns a new instance of Command with clustering enabled.
-func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
+func NewCommandNode(tb testing.TB, isCoordinator bool, opts ...server.CommandOption) *Command {
 	// We want tests to default to using the in-memory translate store, so we
 	// prepend opts with that functional option. If a different translate store
 	// has been specified, it will override this one.
 	opts = prependTestServerOpts(opts)
-	m := newCommand(opts...)
+	m := newCommand(tb, opts...)
 	m.Config.Cluster.Disabled = false
 	m.Config.Cluster.Coordinator = isCoordinator
 	return m
@@ -99,7 +100,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
 // RunCommand returns a new, running Main. Panic on error.
 func RunCommand(t *testing.T) *Command {
 	t.Helper()
-	m := newCommand(server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
+	m := newCommand(t, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)))
 	m.Config.Metric.Diagnostics = false // Disable diagnostics.
 	m.Config.Gossip.Port = "0"
 	if err := m.Start(); err != nil {
@@ -307,8 +308,15 @@ func Do(t *testing.T, method, urlStr string, body string) *httpResponse {
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 
-	resp, err := gohttp.DefaultClient.Do(req)
+	// set a timeout instead of allowing gohttp.Defaultclient to
+	// potentially hang forever.
+	hc := &gohttp.Client{
+		Timeout: time.Second * 10,
+	}
+	resp, err := hc.Do(req)
+
 	if err != nil {
+		fmt.Printf(" hc.Do() err = '%v'\n", err)
 		t.Fatal(err)
 	}
 	defer resp.Body.Close()

test/pilosa_test.go

@@ -30,15 +30,15 @@ func TestNewCluster(t *testing.T) {
 	cluster := test.MustRunCluster(t, numNodes)
 	defer cluster.Close()
 
-	coordinator := getCoordinator(cluster[0])
+	coordinator := getCoordinator(cluster.Nodes[0])
 	for i := 1; i < numNodes; i++ {
-		if coordi := getCoordinator(cluster[i]); coordi != coordinator {
+		if coordi := getCoordinator(cluster.Nodes[i]); coordi != coordinator {
 			t.Fatalf("node %d does not have the same coordinator as node 0. '%v' and '%v' respectively", i, coordi, coordinator)
 		}
 	}
 	req, err := http.NewRequest(
 		"GET",
-		cluster[0].URL()+"/status",
+		cluster.Nodes[0].URL()+"/status",
 		strings.NewReader(""),
 	)
 	if err != nil {

testhook/auditor.go

@@ -0,0 +1,168 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testhook
+
+import (
+	"fmt"
+	"reflect"
+	"sync"
+)
+
+// Auditor represents a thing which knows how to audit events. For instance,
+// it can check on when things are accessed, or when they were opened, or
+// whether opened objects are later closed.
+type Auditor interface {
+
+	// Registry yields a registry for objects of this type.
+	// multiple calls with the same object type yield the same registry.
+	Registry(interface{}) (Registry, error)
+	// Check performs any error-checking that can be done during
+	// usage.
+	Check() (error, []error)
+	// FinalCheck performs any error-checking that makes sense only
+	// after all operations are supposed to be complete, such as
+	// verifying that opened objects have been closed.
+	FinalCheck() (error, []error)
+}
+
+// Created(a, o, kv) is shorthand for a.Registry(o).Created(o, kv) plus
+// the error checking inside that.
+func Created(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Created(o, kv)
+}
+
+// Opened(a, o, kv) is shorthand for a.Registry(o).Opened(o, kv) plus
+// the error checking inside that.
+func Opened(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Opened(o, kv)
+}
+
+// Closed(a, o, kv) is shorthand for a.Registry(o).Closed(o, kv) plus
+// the error checking inside that.
+func Closed(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Closed(o, kv)
+}
+
+// Destroyed(a, o, kv) is shorthand for a.Registry(o).Destroyed(o, kv) plus
+// the error checking inside that.
+func Destroyed(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Destroyed(o, kv)
+}
+
+// Seen(a, o, kv) is shorthand for a.Registry(o).Seen(o, kv) plus
+// the error checking inside that.
+func Seen(a Auditor, o interface{}, kv KV) error {
+	r, err := a.Registry(o)
+	if err != nil {
+		return err
+	}
+	return r.Seen(o, kv)
+}
+
+// NopAuditor doesn't do anything.
+type NopAuditor struct{}
+
+func (*NopAuditor) Registry(interface{}) (Registry, error) {
+	return NewNopRegistry(), nil
+}
+
+func (*NopAuditor) Check() (error, []error) {
+	return nil, nil
+}
+
+func (*NopAuditor) FinalCheck() (error, []error) {
+	return nil, nil
+}
+
+func NewNopAuditor() *NopAuditor {
+	return &NopAuditor{}
+}
+
+// VerifyCloseAuditor provides registries which it will check for things
+// being closed.
+type VerifyCloseAuditor struct {
+	registries map[reflect.Type]Registry
+	hooks      RegistryHooks
+	regMu      sync.Mutex
+}
+
+func (v *VerifyCloseAuditor) Registry(o interface{}) (Registry, error) {
+	t := reflect.TypeOf(o)
+	v.regMu.Lock()
+	defer v.regMu.Unlock()
+	if exists, ok := v.registries[t]; ok {
+		return exists, nil
+	}
+	reg := NewSimpleRegistry(v.hooks[t])
+	v.registries[t] = reg
+	return reg, nil
+}
+
+func (*VerifyCloseAuditor) Check() (error, []error) {
+	return nil, nil
+}
+
+func (v *VerifyCloseAuditor) FinalCheck() (error, []error) {
+	v.regMu.Lock()
+	defer v.regMu.Unlock()
+	var errs []error
+	for t, reg := range v.registries {
+		typeName := t.String()
+		live, err := reg.Live()
+		if err != nil {
+			errs = append(errs, fmt.Errorf("registry[%s]: retrieving live list: %v",
+				typeName, err))
+			continue
+		}
+		if len(live) > 0 {
+			for addr, entry := range live {
+				if entry.Error != nil {
+					errs = append(errs, fmt.Errorf("%v: item created at %v, stack %s",
+						entry.Error, entry.Stamp, entry.Stack))
+				} else {
+					errs = append(errs, fmt.Errorf("live item found at %p, created at %v, stack %s",
+						addr, entry.Stamp, entry.Stack))
+				}
+				if entry.Data["stack"] != nil {
+					errs = append(errs, fmt.Errorf("stashed stack: %s", entry.Data["stack"]))
+				}
+			}
+		}
+	}
+	if len(errs) > 0 {
+		return fmt.Errorf("final check: %d error(s)", len(errs)), errs
+	}
+	return nil, nil
+}
+
+func NewVerifyCloseAuditor(hooks RegistryHooks) *VerifyCloseAuditor {
+	return &VerifyCloseAuditor{registries: map[reflect.Type]Registry{}, hooks: hooks}
+}

testhook/auditor_test.go

@@ -0,0 +1,331 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testhook_test
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa/v2/testhook"
+)
+
+func TestAuditor_CatchError(t *testing.T) {
+	auditor := testhook.NewVerifyCloseAuditor(nil)
+	var x, y int
+	reg, err := auditor.Registry(&x)
+	if err != nil {
+		t.Fatalf("requesting registry: %v", err)
+	}
+	err = reg.Created(&x, nil)
+	if err != nil {
+		t.Fatalf("creating x: %v", err)
+	}
+	err = reg.Opened(&x, nil)
+	if err != nil {
+		t.Fatalf("opening x: %v", err)
+	}
+	err = reg.Seen(&y, nil)
+	if err == nil {
+		t.Fatalf("seeing unopened y: expected error, didn't get one")
+	}
+	err = reg.Seen(&x, nil)
+	if err != nil {
+		t.Fatalf("seeing opened x: unexpected error %v", err)
+	}
+	err = reg.Created(&y, nil)
+	if err != nil {
+		t.Fatalf("creating y: %v", err)
+	}
+	err = reg.Opened(&y, nil)
+	if err != nil {
+		t.Fatalf("opening y: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err != nil {
+		t.Fatalf("closing x: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err == nil {
+		t.Fatalf("double-closing x: expected error, didn't get one")
+	}
+	err = reg.Destroyed(&x, nil)
+	if err != nil {
+		t.Fatalf("destroying x: got unexpected error %v", err)
+	}
+	err, errs := auditor.FinalCheck()
+	if err == nil {
+		t.Fatalf("unclosed y not detected")
+	}
+	_ = errs
+}
+
+type ignoreLiveness struct {
+	skippable *int
+}
+
+func (ign *ignoreLiveness) Live(o interface{}, _ *testhook.RegistryEntry) error {
+	if ptr, ok := o.(*int); ok {
+		if ptr == ign.skippable {
+			return nil
+		}
+	}
+	return errors.New("unexpected live object")
+}
+
+var _ testhook.RegistryHookLive = &ignoreLiveness{}
+
+func TestAuditor_DiscardError(t *testing.T) {
+	var x, y int
+	iptr := reflect.TypeOf(&x)
+	auditor := testhook.NewVerifyCloseAuditor(testhook.RegistryHooks{iptr: &ignoreLiveness{skippable: &y}})
+	reg, err := auditor.Registry(&x)
+	if err != nil {
+		t.Fatalf("requesting registry: %v", err)
+	}
+	err = reg.Created(&x, nil)
+	if err != nil {
+		t.Fatalf("creating x: %v", err)
+	}
+	err = reg.Opened(&x, nil)
+	if err != nil {
+		t.Fatalf("opening x: %v", err)
+	}
+	err = reg.Seen(&y, nil)
+	if err == nil {
+		t.Fatalf("seeing unopened y: expected error, didn't get one")
+	}
+	err = reg.Seen(&x, nil)
+	if err != nil {
+		t.Fatalf("seeing opened x: unexpected error %v", err)
+	}
+	err = reg.Created(&y, nil)
+	if err != nil {
+		t.Fatalf("creating y: %v", err)
+	}
+	err = reg.Opened(&y, nil)
+	if err != nil {
+		t.Fatalf("opening y: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err != nil {
+		t.Fatalf("closing x: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err == nil {
+		t.Fatalf("double-closing x: expected error, didn't get one")
+	}
+	err = reg.Destroyed(&x, nil)
+	if err != nil {
+		t.Fatalf("destroying x: got unexpected error %v", err)
+	}
+	err, errs := auditor.FinalCheck()
+	if err != nil {
+		t.Fatalf("expected to skip y, instead got err %v, error list %v", err, errs)
+	}
+}
+
+func TestAuditor_KeepError(t *testing.T) {
+	var x, y, z int
+	iptr := reflect.TypeOf(&x)
+	auditor := testhook.NewVerifyCloseAuditor(testhook.RegistryHooks{iptr: &ignoreLiveness{skippable: &z}})
+	reg, err := auditor.Registry(&x)
+	if err != nil {
+		t.Fatalf("requesting registry: %v", err)
+	}
+	err = reg.Created(&x, nil)
+	if err != nil {
+		t.Fatalf("creating x: %v", err)
+	}
+	err = reg.Opened(&x, nil)
+	if err != nil {
+		t.Fatalf("opening x: %v", err)
+	}
+	err = reg.Seen(&y, nil)
+	if err == nil {
+		t.Fatalf("seeing unopened y: expected error, didn't get one")
+	}
+	err = reg.Seen(&x, nil)
+	if err != nil {
+		t.Fatalf("seeing opened x: unexpected error %v", err)
+	}
+	err = reg.Created(&y, nil)
+	if err != nil {
+		t.Fatalf("creating y: %v", err)
+	}
+	err = reg.Opened(&y, nil)
+	if err != nil {
+		t.Fatalf("opening y: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err != nil {
+		t.Fatalf("closing x: %v", err)
+	}
+	err = reg.Closed(&x, nil)
+	if err == nil {
+		t.Fatalf("double-closing x: expected error, didn't get one")
+	}
+	err = reg.Destroyed(&x, nil)
+	if err != nil {
+		t.Fatalf("destroying x: %v", err)
+	}
+	err, errs := auditor.FinalCheck()
+	if err == nil {
+		t.Fatalf("undestroyed y not detected")
+	}
+	_ = errs
+}
+
+type failHook struct {
+	calls int
+}
+
+func (f *failHook) Opened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+func (f *failHook) WasOpened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+// Implement WasSeen but not Seen, so we can verify that the only-one
+// case works in both directions.
+func (f *failHook) WasSeen(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+func (f *failHook) Closed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+func (f *failHook) WasClosed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+func (f *failHook) Live(i interface{}, ent *testhook.RegistryEntry) error {
+	f.calls++
+	return errors.New("failHook always fails")
+}
+
+type successHook struct {
+	calls int
+}
+
+func (s *successHook) Opened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	s.calls++
+	return nil
+}
+
+func (s *successHook) WasOpened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	s.calls++
+	return nil
+}
+
+func (s *successHook) Closed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	s.calls++
+	return nil
+}
+
+func (s *successHook) Seen(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error {
+	s.calls++
+	return nil
+}
+
+// successHook allows an error to leak from WasClosed.
+func (s *successHook) WasClosed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error {
+	s.calls++
+	return err
+}
+
+func (s *successHook) Live(i interface{}, ent *testhook.RegistryEntry) error {
+	s.calls++
+	return errors.New("even successHook can fail sometimes")
+}
+
+func TestComposedHooks(t *testing.T) {
+	s := &successHook{}
+	f := &failHook{}
+	sExp, fExp := 0, 0
+	var err error
+	checkExp := func(when string) {
+		if s.calls != sExp {
+			t.Fatalf("after %s, expected %d calls to successHook, got %d", when, sExp, s.calls)
+		}
+		if f.calls != fExp {
+			t.Fatalf("after %s, expected %d calls to failHook, got %d", when, fExp, f.calls)
+		}
+	}
+	combined := testhook.Compose(s, f)
+	// Opened: we expect both opened calls to be hit, and the error from
+	// the second to come back.
+	err = combined.Opened(nil, nil, nil)
+	sExp++
+	fExp++
+	checkExp("opened")
+	if err == nil {
+		t.Fatalf("composed hook, opened: didn't error")
+	}
+	if err.Error() != "failHook always fails" {
+		t.Fatalf("composed hook, opened: expected failHook always fails, got %v", err)
+	}
+
+	// WasOpened: we expect the error from failHook to be overridden.
+	err = combined.WasOpened(nil, nil, nil, nil)
+	sExp++
+	fExp++
+	checkExp("wasOpened")
+	if err != nil {
+		t.Fatalf("composed hook, wasOpened: expected no error, got %v", err)
+	}
+
+	// Seen: nothing to call for failHook
+	err = combined.Seen(nil, nil, nil)
+	sExp++
+	checkExp("seen")
+	if err != nil {
+		t.Fatalf("composed hook, seen: expected no error, got %v", err)
+	}
+
+	// WasSeen: nothing to call for successHook
+	err = combined.WasSeen(nil, nil, nil, nil)
+	fExp++
+	checkExp("wasSeen")
+	if err == nil {
+		t.Fatalf("composed hook, wasSeen: expected error, didn't get it")
+	}
+
+	// WasClosed: expect both to get called, but successHook to leak the error up
+	err = combined.WasClosed(nil, nil, nil, nil)
+	sExp++
+	fExp++
+	checkExp("wasClosed")
+	if err == nil {
+		t.Fatalf("composed hook, wasClosed: expected error, didn't get it")
+	}
+
+	// Live: the failure from successHook (oops) should prevent failHook from
+	// being called.
+	err = combined.Live(nil, nil)
+	sExp++
+	checkExp("live")
+	if err == nil {
+		t.Fatalf("composed hook, live: expected error, didn't get it")
+	}
+}

testhook/cleanup1.13.go

@@ -0,0 +1,47 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !go1.14
+
+package testhook
+
+import (
+	"sync"
+	"testing"
+)
+
+var cleanupFuncs []func()
+var cleanupMu sync.Mutex
+
+func init() {
+	RegisterPostTestHook(runCleanupFuncs)
+}
+
+func runCleanupFuncs() error {
+	cleanupMu.Lock()
+	defer cleanupMu.Unlock()
+	for _, fn := range cleanupFuncs {
+		fn()
+	}
+	return nil
+}
+
+// Cleanup in 1.13 logs a message about skipping a cleanup function, but
+// allows things to build. Cleanup in 1.14 uses tb.Cleanup to register
+// a cleanup function to call when a test completes.
+func Cleanup(tb testing.TB, fn func()) {
+	cleanupMu.Lock()
+	defer cleanupMu.Unlock()
+	cleanupFuncs = append(cleanupFuncs, fn)
+}

testhook/cleanup1.14.go

@@ -0,0 +1,28 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.14
+
+package testhook
+
+import (
+	"testing"
+)
+
+// Cleanup in 1.13 logs a message about skipping a cleanup function, but
+// allows things to build. Cleanup in 1.14 uses tb.Cleanup to register
+// a cleanup function to call when a test completes.
+func Cleanup(tb testing.TB, fn func()) {
+	tb.Cleanup(fn)
+}

testhook/hook.go

@@ -0,0 +1,107 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testhook
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sync"
+	"testing"
+)
+
+// Callback denotes a function which can be run on a testing.T, or testing.B,
+// which performs additional functions typically before or after tests.
+type Callback func() error
+
+var preHooks []Callback
+var postHooks []Callback
+var mu sync.Mutex
+
+// RegisterPostTestHook registers a function to be called after tests
+// are run. It should return a nil error if it's okay, and a non-nil
+// error to cause a non-zero exit status.
+func RegisterPostTestHook(fn Callback) {
+	mu.Lock()
+	defer mu.Unlock()
+	postHooks = append(postHooks, fn)
+}
+
+// RegisterPreTestHook registers a function to be called after tests
+// are run. It should return a nil error if it's okay, and a non-nil
+// error to cause a non-zero exit status.
+func RegisterPreTestHook(fn Callback) {
+	mu.Lock()
+	defer mu.Unlock()
+	preHooks = append(preHooks, fn)
+}
+
+// RunTestsWithHooks is a suitable implementation for TestMain; you can
+// just invoke this from your TestMain, passing in m, and it runs the tests
+// and then runs any registered pre/post hooks. If the hooks themselves try
+// to register hooks, you will deadlock. Don't do that.
+func RunTestsWithHooks(m *testing.M) {
+	var ret int
+	mu.Lock()
+	for _, fn := range preHooks {
+		err := fn()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "pre-hook failure: %v\n", err)
+			ret = 1
+		}
+	}
+	mu.Unlock()
+	if ret != 0 {
+		fmt.Fprint(os.Stderr, "pre-hooks failed, aborting.\n")
+		os.Exit(ret)
+	}
+	ret = m.Run()
+	mu.Lock()
+	defer mu.Unlock()
+	for _, fn := range postHooks {
+		err := fn()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "post-hook failure: %v\n", err)
+			ret = 1
+		}
+	}
+	os.Exit(ret)
+}
+
+// TempDir creates a temp directory that will be automatically deleted when
+// this test completes, using go1.14's [TB].Cleanup() if available.
+func TempDir(tb testing.TB, pattern string) (path string, err error) {
+	path, err = ioutil.TempDir("", pattern)
+	if err == nil {
+		Cleanup(tb, func() {
+			os.RemoveAll(path)
+		})
+	}
+	return path, err
+}
+
+// TempDirInDir creates a temp directory that will be automatically deleted when
+// this test completes, using go1.14's [TB].Cleanup(), but with a specified
+// path instead of the default Go TMPDIR. Only some tests use this, which is
+// possibly an error...
+func TempDirInDir(tb testing.TB, dir string, pattern string) (path string, err error) {
+	path, err = ioutil.TempDir(dir, pattern)
+	if err == nil {
+		Cleanup(tb, func() {
+			os.RemoveAll(path)
+		})
+	}
+	return path, err
+}

testhook/registry.go

@@ -0,0 +1,524 @@
+// Copyright 2020 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testhook
+
+import (
+	"fmt"
+	"reflect"
+	"runtime/debug"
+	"sync"
+	"time"
+)
+
+// KV represents a key/value mapping. You don't need to use the type
+// for anything, it's just to make typing easier.
+type KV map[string]interface{}
+
+// Registry represents a set of known objects of a given common type.
+// A typical implementation might maintain a map of objects it's seen which
+// haven't been deleted. A minimal implementation just does nothing.
+//
+// A Registry may implement reference-count semantics, or may treat
+// reopening of a still-open object as an error.
+//
+// All objects provided to a registry should have the same type.
+type Registry interface {
+	Created(interface{}, KV) error                 // Object has been created, and must be destroyed later.
+	Opened(interface{}, KV) error                  // Object has been opened, and must be closed later.
+	Seen(interface{}, KV) error                    // Object has been interacted with, and should be between an open and a close.
+	Closed(interface{}, KV) error                  // Object has been closed, and must have been opened previously.
+	Destroyed(interface{}, KV) error               // Object has been destroyed
+	Live() (map[interface{}]*RegistryEntry, error) // Currently live objects
+}
+
+// RegistryHook is a generic interface, but any real implementation should
+// implement at least one of RegistryHookPreOpen, RegistryHookPostOpen,
+// etcetera. Pre-hooks are called before any error checks by the registry,
+// post-hooks are called with after those error checks, and are passed the
+// (possibly nil) error that would be returned at that point. If the post-hook
+// overrides the error, the registry continues with operations as though it
+// hadn't occurred, which may be a very bad idea.
+type RegistryHook interface{}
+
+type RegistryHookPreCreate interface {
+	Created(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostCreate interface {
+	WasCreated(interface{}, KV, *RegistryEntry, error) error
+}
+
+type RegistryHookPreOpen interface {
+	Opened(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostOpen interface {
+	WasOpened(interface{}, KV, *RegistryEntry, error) error
+}
+
+type RegistryHookPreSee interface {
+	Seen(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostSee interface {
+	WasSeen(interface{}, KV, *RegistryEntry, error) error
+}
+
+type RegistryHookPreClose interface {
+	Closed(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostClose interface {
+	WasClosed(interface{}, KV, *RegistryEntry, error) error
+}
+
+type RegistryHookPreDestroy interface {
+	Destroyed(interface{}, KV, *RegistryEntry) error
+}
+
+type RegistryHookPostDestroy interface {
+	WasDestroyed(interface{}, KV, *RegistryEntry, error) error
+}
+
+// If a registry's hooks implement RegistryHookLive, Live() should
+// return only those entries for which a non-nil error was returned, with the
+// error inserted in the RegistryEntry.
+type RegistryHookLive interface {
+	Live(interface{}, *RegistryEntry) error
+}
+
+// RegistryHooks represents a set of registry hook values to use for
+// registries, corresponding to different types.
+type RegistryHooks map[reflect.Type]RegistryHook
+
+// RegistryEntry represents the data we might have about an entry. Every
+// entry in it could be zero-valued in some implementations
+type RegistryEntry struct {
+	Error     error
+	Stack     []byte
+	Stamp     time.Time
+	Data      KV
+	OpenCount int
+}
+
+// NopRegistry doesn't do anything; it exists to fit the interface but not
+// consume resources.
+type NopRegistry struct{}
+
+var _ Registry = &NopRegistry{}
+
+func (*NopRegistry) Created(interface{}, KV) error                 { return nil }
+func (*NopRegistry) Opened(interface{}, KV) error                  { return nil }
+func (*NopRegistry) Seen(interface{}, KV) error                    { return nil }
+func (*NopRegistry) Closed(interface{}, KV) error                  { return nil }
+func (*NopRegistry) Destroyed(interface{}, KV) error               { return nil }
+func (*NopRegistry) Live() (map[interface{}]*RegistryEntry, error) { return nil, nil }
+
+func NewNopRegistry() *NopRegistry {
+	return &NopRegistry{}
+}
+
+// SimpleRegistry asserts that objects are created, then opened, then
+// possibly seen, then closed, then destroyed, and that they are not
+// opened more than once at a time, or destroyed while open. As a
+// convenience feature for users whose use cases might rely on this,
+// it will actually accept a new item being opened without being
+// previously created; to prevent this, use a PreOpen hook that checks
+// for a nil *RegistryEntry. The default Live check will report only
+// objects with an open count other than 0, but if you provide a Live
+// hook, you can return errors for objects still existing.
+type SimpleRegistry struct {
+	hooks   RegistryHook
+	entries map[interface{}]*RegistryEntry
+	mu      sync.Mutex
+}
+
+var _ Registry = &SimpleRegistry{}
+
+func NewSimpleRegistry(hooks RegistryHook) *SimpleRegistry {
+	return &SimpleRegistry{entries: map[interface{}]*RegistryEntry{}, hooks: hooks}
+}
+
+func (s *SimpleRegistry) Created(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreCreate); ok {
+		if err := hook.Created(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing != nil {
+		err = fmt.Errorf("object %T:%v previously registered at %v", o, o, existing.Stamp)
+	} else {
+		existing = &RegistryEntry{
+			Stack: debug.Stack(),
+			Data:  kv,
+			Stamp: time.Now(),
+		}
+	}
+	if hook, ok := s.hooks.(RegistryHookPostCreate); ok {
+		err = hook.WasCreated(o, kv, existing, err)
+	}
+	if err != nil {
+		return err
+	}
+	// if you overrode the error, or there wasn't one and you didn't
+	// create one, we now stash the entry.
+	s.entries[o] = existing
+	return nil
+}
+
+func (s *SimpleRegistry) Opened(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreOpen); ok {
+		if err := hook.Opened(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing == nil {
+		existing = &RegistryEntry{
+			Stack: debug.Stack(),
+			Data:  kv,
+			Stamp: time.Now(),
+		}
+		s.entries[o] = existing
+	}
+	existing.OpenCount++
+	if existing.OpenCount > 1 {
+		err = fmt.Errorf("object %T:%v opened %d times", o, o, existing.OpenCount)
+	}
+	if hook, ok := s.hooks.(RegistryHookPostOpen); ok {
+		err = hook.WasOpened(o, kv, existing, err)
+	}
+	if err != nil {
+		if existing != nil {
+			existing.OpenCount--
+		}
+		return err
+	}
+	return nil
+}
+
+func (s *SimpleRegistry) Seen(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreSee); ok {
+		if err = hook.Seen(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing == nil {
+		err = fmt.Errorf("object %T:%v seen but not previously registered", o, o)
+	}
+	if hook, ok := s.hooks.(RegistryHookPostSee); ok {
+		err = hook.WasSeen(o, kv, existing, err)
+	}
+	return err
+}
+
+func (s *SimpleRegistry) Closed(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreClose); ok {
+		if err = hook.Closed(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing == nil {
+		err = fmt.Errorf("object %T:%v closed but not previously registered", o, o)
+	} else {
+		existing.OpenCount--
+		if existing.OpenCount < 0 {
+			err = fmt.Errorf("object %T:%v closed more often than it was open: %d", o, o, existing.OpenCount)
+		}
+	}
+	if hook, ok := s.hooks.(RegistryHookPostClose); ok {
+		err = hook.WasClosed(o, kv, existing, err)
+	}
+	if err != nil {
+		// if a close "failed", we don't want to count it as being closed.
+		if existing != nil {
+			existing.OpenCount++
+		}
+		return err
+	}
+	return nil
+}
+
+func (s *SimpleRegistry) Destroyed(o interface{}, kv KV) (err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	existing := s.entries[o]
+	if hook, ok := s.hooks.(RegistryHookPreDestroy); ok {
+		if err = hook.Destroyed(o, kv, existing); err != nil {
+			return err
+		}
+	}
+	if existing == nil {
+		err = fmt.Errorf("object %T:%v destroyed but not previously registered", o, o)
+	} else if existing.OpenCount != 0 {
+		err = fmt.Errorf("object %T:%v destroyed while open count is %d", o, o, existing.OpenCount)
+	}
+	if hook, ok := s.hooks.(RegistryHookPostDestroy); ok {
+		err = hook.WasDestroyed(o, kv, existing, err)
+	}
+	if err != nil {
+		fmt.Printf("destroyed error: %v\n", err)
+		return err
+	}
+	// remove the entry from the list.
+	delete(s.entries, o)
+	return nil
+}
+
+func (s *SimpleRegistry) Live() (results map[interface{}]*RegistryEntry, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	results = make(map[interface{}]*RegistryEntry)
+	if hook, ok := s.hooks.(RegistryHookLive); ok {
+		for k, v := range s.entries {
+			v.Error = hook.Live(k, v)
+			if v.Error != nil {
+				results[k] = v
+			}
+		}
+	} else {
+		for k, v := range s.entries {
+			if v.OpenCount != 0 {
+				v.Error = fmt.Errorf("open count %d", v.OpenCount)
+				results[k] = v
+			}
+		}
+	}
+	return results, nil
+}
+
+type PreHook func(interface{}, KV, *RegistryEntry) error
+type PostHook func(interface{}, KV, *RegistryEntry, error) error
+type LiveHook func(interface{}, *RegistryEntry) error
+
+// PluggableRegistry is a registry which takes dynamically-generated functions,
+// and calls them if they're not nil.
+type PluggableRegistry struct {
+	implCreated      PreHook
+	implWasCreated   PostHook
+	implOpened       PreHook
+	implWasOpened    PostHook
+	implSeen         PreHook
+	implWasSeen      PostHook
+	implClosed       PreHook
+	implWasClosed    PostHook
+	implDestroyed    PreHook
+	implWasDestroyed PostHook
+	implLive         LiveHook
+}
+
+func (p *PluggableRegistry) Created(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implCreated != nil {
+		return p.implCreated(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasCreated(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasCreated != nil {
+		return p.implWasCreated(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Opened(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implOpened != nil {
+		return p.implOpened(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasOpened(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasOpened != nil {
+		return p.implWasOpened(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Seen(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implSeen != nil {
+		return p.implSeen(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasSeen(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasSeen != nil {
+		return p.implWasSeen(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Closed(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implClosed != nil {
+		return p.implClosed(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasClosed(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasClosed != nil {
+		return p.implWasClosed(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Destroyed(i interface{}, kv KV, ent *RegistryEntry) error {
+	if p.implDestroyed != nil {
+		return p.implDestroyed(i, kv, ent)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) WasDestroyed(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+	if p.implWasDestroyed != nil {
+		return p.implWasDestroyed(i, kv, ent, err)
+	}
+	return nil
+}
+
+func (p *PluggableRegistry) Live(i interface{}, ent *RegistryEntry) error {
+	if p.implLive != nil {
+		return p.implLive(i, ent)
+	}
+	return nil
+}
+
+func composePreHooks(fns ...PreHook) PreHook {
+	if len(fns) == 0 {
+		return nil
+	}
+	if len(fns) == 1 {
+		return fns[0]
+	}
+	return func(i interface{}, kv KV, ent *RegistryEntry) error {
+		for _, fn := range fns {
+			err := fn(i, kv, ent)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+func composePostHooks(fns ...PostHook) PostHook {
+	if len(fns) == 0 {
+		return nil
+	}
+	if len(fns) == 1 {
+		return fns[0]
+	}
+	return func(i interface{}, kv KV, ent *RegistryEntry, err error) error {
+		// We run the functions in reverse order, so the last
+		// hook added has the option of overriding a lower hook's
+		// opinion.
+		for i := range fns {
+			fn := fns[len(fns)-1-i]
+			err = fn(i, kv, ent, err)
+		}
+		return err
+	}
+}
+
+func composeLiveHooks(fns ...LiveHook) LiveHook {
+	if len(fns) == 0 {
+		return nil
+	}
+	if len(fns) == 1 {
+		return fns[0]
+	}
+	return func(i interface{}, ent *RegistryEntry) error {
+		for _, fn := range fns {
+			err := fn(i, ent)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+// Compose takes a list of RegistryHook objects, and combines their hook
+// functions into a unified registry. For Pre hooks and Live hooks, the
+// functions are called in the order they were provided to this function,
+// and the first one to return an error causes the remainder to be
+// skipped; for Post hooks, they are called in the opposite order, and
+// the whole list continues to be called, with each function getting passed
+// the error (or non-error) value returned by the previous one.
+func Compose(hooks ...RegistryHook) *PluggableRegistry {
+	var created, opened, seen, closed, destroyed []PreHook
+	var wasCreated, wasOpened, wasSeen, wasClosed, wasDestroyed []PostHook
+	var live []LiveHook
+	for _, h := range hooks {
+		if hook, ok := h.(RegistryHookPreCreate); ok {
+			created = append(created, hook.Created)
+		}
+		if hook, ok := h.(RegistryHookPostCreate); ok {
+			wasCreated = append(wasCreated, hook.WasCreated)
+		}
+		if hook, ok := h.(RegistryHookPreOpen); ok {
+			opened = append(opened, hook.Opened)
+		}
+		if hook, ok := h.(RegistryHookPostOpen); ok {
+			wasOpened = append(wasOpened, hook.WasOpened)
+		}
+		if hook, ok := h.(RegistryHookPreSee); ok {
+			seen = append(seen, hook.Seen)
+		}
+		if hook, ok := h.(RegistryHookPostSee); ok {
+			wasSeen = append(wasSeen, hook.WasSeen)
+		}
+		if hook, ok := h.(RegistryHookPreClose); ok {
+			closed = append(closed, hook.Closed)
+		}
+		if hook, ok := h.(RegistryHookPostClose); ok {
+			wasClosed = append(wasClosed, hook.WasClosed)
+		}
+		if hook, ok := h.(RegistryHookPreDestroy); ok {
+			destroyed = append(destroyed, hook.Destroyed)
+		}
+		if hook, ok := h.(RegistryHookPostDestroy); ok {
+			wasDestroyed = append(wasDestroyed, hook.WasDestroyed)
+		}
+		if hook, ok := h.(RegistryHookLive); ok {
+			live = append(live, hook.Live)
+		}
+	}
+	return &PluggableRegistry{
+		implCreated:      composePreHooks(created...),
+		implWasCreated:   composePostHooks(wasCreated...),
+		implOpened:       composePreHooks(opened...),
+		implWasOpened:    composePostHooks(wasOpened...),
+		implSeen:         composePreHooks(seen...),
+		implWasSeen:      composePostHooks(wasSeen...),
+		implClosed:       composePreHooks(closed...),
+		implWasClosed:    composePostHooks(wasClosed...),
+		implDestroyed:    composePreHooks(destroyed...),
+		implWasDestroyed: composePostHooks(wasDestroyed...),
+		implLive:         composeLiveHooks(live...),
+	}
+}

translator_test.go

@@ -227,11 +227,12 @@ func TestTranslation_Reset(t *testing.T) {
 					pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
 				)},
 		)
+		defer c.Close()
 
-		node0 := c[0]
-		node1 := c[1]
-		node2 := c[2]
-		node3 := c[3]
+		node0 := c.GetNode(0)
+		node1 := c.GetNode(1)
+		node2 := c.GetNode(2)
+		node3 := c.GetNode(3)
 
 		ctx := context.Background()
 		idx := "i"
@@ -321,9 +322,10 @@ func TestTranslation_Replication(t *testing.T) {
 					pilosa.OptServerReplicaN(2),
 				)},
 		)
+		defer c.Close()
 
-		node0 := c[0]
-		node1 := c[1]
+		node0 := c.GetNode(0)
+		node1 := c.GetNode(1)
 
 		ctx := context.Background()
 		idx := "i"
@@ -362,7 +364,7 @@ func TestTranslation_Replication(t *testing.T) {
 		node0.QueryExpect(t, idx, "", `Row(f=1)`, exp)
 
 		// Kill one node.
-		if err := node1.Command.Close(); err != nil {
+		if err := c.CloseAndRemove(1); err != nil {
 			t.Fatal(err)
 		}
 
@@ -396,8 +398,8 @@ func TestTranslation_Coordinator(t *testing.T) {
 		)
 		defer c.Close()
 
-		node0 := c[0]
-		node1 := c[1]
+		node0 := c.GetNode(0)
+		node1 := c.GetNode(1)
 
 		ctx := context.Background()
 		idx := "i"

tx_test.go

@@ -58,6 +58,7 @@ func queryBalances(m0api *pilosa.API, acctOwnerID uint64, fldAcct0, fldAcct1, in
 	acct1bal = mustQueryAcct(m0api, acctOwnerID, fldAcct1, index)
 	return
 }
+
 func skipForRoaring(t *testing.T) {
 	src := os.Getenv("PILOSA_TXSRC")
 	// once txfactory.go DefaultTxsrc != RoaringTxn, this
@@ -67,7 +68,7 @@ func skipForRoaring(t *testing.T) {
 	}
 }
 
-func TestAPI_ImportAIR(t *testing.T) {
+func TestAPI_ImportAtomicRecord(t *testing.T) {
 	skipForRoaring(t)
 	c := test.MustRunCluster(t, 1,
 		[]server.CommandOption{
@@ -79,7 +80,7 @@ func TestAPI_ImportAIR(t *testing.T) {
 	)
 	defer c.Close()
 
-	m0 := c[0]
+	m0 := c.GetNode(0)
 	m0api := m0.API
 
 	ctx := context.Background()

utils_internal_test.go

@@ -19,15 +19,17 @@ import (
 	"io/ioutil"
 	"path/filepath"
 	"sync"
+	"testing"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 )
 
 // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
-func NewTestCluster(n int) *cluster {
-	path, err := ioutil.TempDir("", "pilosa-cluster-")
+func NewTestCluster(tb testing.TB, n int) *cluster {
+	path, err := testhook.TempDir(tb, "pilosa-cluster-")
 	if err != nil {
 		panic(err)
 	}
@@ -88,6 +90,7 @@ type ClusterCluster struct {
 	mu         sync.RWMutex
 	resizing   bool
 	resizeDone chan struct{}
+	tb         testing.TB
 }
 
 type commonClusterSettings struct {
@@ -226,7 +229,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error)
 	t.common.Nodes = append(t.common.Nodes, node)
 
 	// create node-specific temp directory
-	path, err := ioutil.TempDir(*TempDir, fmt.Sprintf("pilosa-cluster-node-%d-", i))
+	path, err := testhook.TempDirInDir(t.tb, *TempDir, fmt.Sprintf("pilosa-cluster-node-%d-", i))
 	if err != nil {
 		return nil, err
 	}
@@ -262,10 +265,11 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error)
 }
 
 // NewClusterCluster returns a new instance of test.Cluster.
-func NewClusterCluster(n int) *ClusterCluster {
+func NewClusterCluster(tb testing.TB, n int) *ClusterCluster {
 
 	tc := &ClusterCluster{
 		common: &commonClusterSettings{},
+		tb:     tb,
 	}
 
 	// add clusters

view.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pilosa/pilosa/v2/pql"
 	"github.com/pilosa/pilosa/v2/roaring"
 	"github.com/pilosa/pilosa/v2/stats"
+	"github.com/pilosa/pilosa/v2/testhook"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 )
@@ -158,6 +159,7 @@ func (v *view) open() error {
 		return err
 	}
 
+	_ = testhook.Opened(v.holder.Auditor, v, nil)
 	v.holder.Logger.Debugf("successfully opened index/field/view: %s/%s/%s", v.index, v.field, v.name)
 	return nil
 }
@@ -224,6 +226,9 @@ shardLoop:
 func (v *view) close() error {
 	v.mu.Lock()
 	defer v.mu.Unlock()
+	defer func() {
+		_ = testhook.Closed(v.holder.Auditor, v, nil)
+	}()
 
 	// Close all fragments.
 	eg, ctx := errgroup.WithContext(context.Background())
@@ -394,6 +399,7 @@ func (v *view) deleteFragment(shard uint64) error {
 	v.holder.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)
 
 	idx := f.holder.Index(v.index)
+	f.Close()
 	if err := idx.Txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil {
 		return errors.Wrap(err, "DeleteFragment")
 	}

view_internal_test.go

@@ -15,16 +15,16 @@
 package pilosa
 
 import (
-	"io/ioutil"
 	"testing"
 	"time"
 
+	"github.com/pilosa/pilosa/v2/testhook"
 	"golang.org/x/sync/errgroup"
 )
 
 // mustOpenView returns a new instance of View with a temporary path.
-func mustOpenView(index, field, name string) *view {
-	path, err := ioutil.TempDir(*TempDir, "pilosa-view-")
+func mustOpenView(tb testing.TB, index, field, name string) *view {
+	path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-view-")
 	if err != nil {
 		panic(err)
 	}
@@ -38,7 +38,9 @@ func mustOpenView(index, field, name string) *view {
 	h.Path = path
 	// h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment
 	idx, err := h.createIndex(index, IndexOptions{})
-	_ = idx
+	testhook.Cleanup(tb, func() {
+		h.Close()
+	})
 	panicOn(err)
 
 	v := newView(h, path, index, field, name, fo)
@@ -54,7 +56,7 @@ func mustOpenView(index, field, name string) *view {
 
 // Ensure view can open and retrieve a fragment.
 func TestView_DeleteFragment(t *testing.T) {
-	v := mustOpenView("i", "f", "v")
+	v := mustOpenView(t, "i", "f", "v")
 	defer v.close()
 
 	shard := uint64(9)
@@ -89,7 +91,7 @@ func TestView_DeleteFragment(t *testing.T) {
 // if the broadcast operation takes a bit of time.
 func TestView_CreateFragmentRace(t *testing.T) {
 	var creates errgroup.Group
-	v := mustOpenView("i", "f", "v")
+	v := mustOpenView(t, "i", "f", "v")
 	defer v.close()
 
 	// Use a broadcaster which intentionally fails.