diff --git a/.circleci/config.yml b/.circleci/config.yml index e0cd85892..16180ae16 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -186,6 +186,7 @@ workflows: matrix: parameters: golang_version: ["1.14", "1.13"] + resource_class: large requires: - setup filters: diff --git a/Makefile b/Makefile index 5794843ce..bd605f1d3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -43,6 +43,33 @@ test: test-race: go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v +testv: topt testvsub + +testv-race: topt-race testvsub-race + +# testvsub: run go test -v in sub-directories in "local mode" with incremental output, +# avoiding go -test ./... "package list mode" which doesn't give output +# until the test run finishes. Package list mode makes it hard to +# find which test is hung/deadlocked. +# +testvsub: + set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \ + echo; echo "___ testing subpkg $$i"; \ + cd $$i; pwd; \ + go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v || break; \ + echo; echo "999 done testing subpkg $$i"; \ + cd ..; \ + done + +testvsub-race: + set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \ + echo; echo "___ testing subpkg $$i -race"; \ + cd $$i; pwd; \ + go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race || break; \ + echo; echo "999 done testing subpkg $$i -race"; \ + cd ..; \ + done + bench: go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS) diff --git a/api.go b/api.go index 8d3bfcea5..05381f8c9 100644 --- a/api.go +++ b/api.go @@ -1114,9 +1114,6 @@ func (api *API) ImportAtomicRecord(ctx context.Context, req *AtomicRecord, opts return tx.Commit() } -// This is a hide your face ugly hack, forced upon -// us by the horrible invention of function based options -// by the usually brilliant Rob Pike. - JEA func addClearToImportOptions(opts []ImportOption) []ImportOption { var opt ImportOptions for _, o := range opts { diff --git a/api_test.go b/api_test.go index 88bb65510..8eb850cbc 100644 --- a/api_test.go +++ b/api_test.go @@ -59,8 +59,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { ) defer c.Close() - m0 := c[0] - m1 := c[1] + m0 := c.GetNode(0) + m1 := c.GetNode(1) t.Run("ImportColumnAttrs", func(t *testing.T) { ctx := context.Background() indexName := "i" @@ -184,8 +184,8 @@ func TestAPI_Import(t *testing.T) { ) defer c.Close() - m0 := c[0] - m1 := c[1] + m0 := c.GetNode(0) + m1 := c.GetNode(1) t.Run("RowIDColumnKey", func(t *testing.T) { ctx := context.Background() @@ -293,8 +293,8 @@ func TestAPI_ImportValue(t *testing.T) { ) defer c.Close() - m0 := c[0] - m1 := c[1] + m0 := c.GetNode(0) + m1 := c.GetNode(1) t.Run("ValColumnKey", func(t *testing.T) { ctx := context.Background() @@ -492,7 +492,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) { // 3. verifiy the clear is done. // repeat for ImportValueRequest and ImportValues() - m0 := c[0] + m0 := c.GetNode(0) m0api := m0.API ctx := context.Background() diff --git a/audit.go b/audit.go new file mode 100644 index 000000000..86be7037e --- /dev/null +++ b/audit.go @@ -0,0 +1,25 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "github.com/pilosa/pilosa/v2/testhook" +) + +var NewAuditor func() testhook.Auditor = NewNopAuditor + +func NewNopAuditor() testhook.Auditor { + return testhook.NewNopAuditor() +} diff --git a/audit_internal_test.go b/audit_internal_test.go new file mode 100644 index 000000000..ab4766e49 --- /dev/null +++ b/audit_internal_test.go @@ -0,0 +1,52 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "fmt" + "reflect" + + "github.com/pilosa/pilosa/v2/testhook" +) + +// These audit hooks are desireable during testing, but not in +// production. +type auditorViewHooks struct{} +type auditorFragmentHooks struct{} + +// static type checks +var _ testhook.RegistryHookLive = &auditorViewHooks{} +var _ testhook.RegistryHookLive = &auditorFragmentHooks{} + +func (*auditorViewHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("view %s still open", o.(*view).name) + } + return nil +} + +func (*auditorFragmentHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("fragment %s still open", o.(*fragment).path) + } + return nil +} + +func GetInternalTestHooks() testhook.RegistryHooks { + return map[reflect.Type]testhook.RegistryHook{ + reflect.TypeOf((*view)(nil)): &auditorViewHooks{}, + reflect.TypeOf((*fragment)(nil)): &auditorFragmentHooks{}, + } +} diff --git a/audit_test.go b/audit_test.go new file mode 100644 index 000000000..1587ded3f --- /dev/null +++ b/audit_test.go @@ -0,0 +1,107 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa_test + +import ( + "fmt" + "os" + "reflect" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/testhook" +) + +// AuditLeaksOn is a global switch to turn on resource +// leak checking at the end of a test run. +var AuditLeaksOn = true + +// for tests, we use a single shared auditor used by all of the holders. +var globalTestAuditor = testhook.NewVerifyCloseAuditor(testHooks) + +// These audit hooks are desireable during testing, but not in +// production. +type auditorIndexHooks struct{} +type auditorFieldHooks struct{} +type auditorHolderHooks struct{} + +// static type checking +var _ testhook.RegistryHookLive = &auditorIndexHooks{} +var _ testhook.RegistryHookLive = &auditorFieldHooks{} +var _ testhook.RegistryHookPostDestroy = &auditorHolderHooks{} +var _ testhook.RegistryHookLive = &auditorHolderHooks{} + +var testHooks = map[reflect.Type]testhook.RegistryHook{ + reflect.TypeOf((*pilosa.Index)(nil)): &auditorIndexHooks{}, + reflect.TypeOf((*pilosa.Field)(nil)): &auditorFieldHooks{}, + reflect.TypeOf((*pilosa.Holder)(nil)): &auditorHolderHooks{}, +} + +func init() { + if !AuditLeaksOn { + return + } + for k, v := range pilosa.GetInternalTestHooks() { + testHooks[k] = v + } + testhook.RegisterPreTestHook(func() error { + pilosa.NewAuditor = NewTestAuditor + return nil + }) + testhook.RegisterPostTestHook(func() error { + err, errs := globalTestAuditor.FinalCheck() + if err != nil { + for i, e := range errs { + fmt.Fprintf(os.Stderr, "[%d]: %v\n", i, e) + } + } + return err + }) +} + +func NewTestAuditor() testhook.Auditor { + return globalTestAuditor +} + +func (*auditorIndexHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("index %s still open", o.(*pilosa.Index).Name()) + } + return nil +} + +func (*auditorFieldHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("field %s still open", o.(*pilosa.Field).Name()) + } + return nil +} + +func (*auditorHolderHooks) WasDestroyed(o interface{}, kv testhook.KV, ent *testhook.RegistryEntry, err error) error { + path := o.(*pilosa.Holder).Path + if path == "" { + fmt.Fprintf(os.Stderr, "OOPS: trying to destroy a holder with no path! created: %s\n", + ent.Stack) + } else { + os.RemoveAll(o.(*pilosa.Holder).Path) + } + return err +} + +func (*auditorHolderHooks) Live(o interface{}, entry *testhook.RegistryEntry) error { + if entry != nil && entry.OpenCount != 0 { + return fmt.Errorf("holder %s still open", o.(*pilosa.Holder).Path) + } + return nil +} diff --git a/badger.go b/badger.go index 3ab0dbb5f..52e1364c2 100644 --- a/badger.go +++ b/badger.go @@ -32,6 +32,7 @@ import ( badger "github.com/dgraph-io/badger/v2" badgeroptions "github.com/dgraph-io/badger/v2/options" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pilosa/pilosa/v2/txkey" "github.com/pkg/errors" ) @@ -329,6 +330,7 @@ func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, e halt: halt, hasher: NewBlake3Hasher(), } + _ = testhook.Opened(NewAuditor(), w, nil) r.unprotectedRegister(w) w.startStack = stack() @@ -504,6 +506,7 @@ func (w *BadgerDBWrapper) Close() (err error) { close(w.halt) w.closed = true } + _ = testhook.Closed(NewAuditor(), w, nil) return w.db.Close() } diff --git a/badger_test.go b/badger_test.go index c1627f1db..ddd67563f 100644 --- a/badger_test.go +++ b/badger_test.go @@ -18,12 +18,8 @@ // See https://github.com/dgraph-io/badger/issues/1384 for any progress. // What we see is that the value-log allocations immediately run out of // memory. So we turn off 386 with a build tag to keep the .circleci happy. -// -// gendebug_test will have a TestMain if build tag generationdebug is on, -// so we avoid conflicting with that debug scenario. // +build !386 -// +build !generationdebug package pilosa @@ -36,10 +32,16 @@ import ( "github.com/dgraph-io/badger/v2" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/testhook" + "github.com/pkg/errors" ) var _ = &roaring.Bitmap{} +func init() { + testhook.RegisterPostTestHook(reportTestBadgersNeedingClose) +} + // helpers, each runs their own new txn, and commits if a change/delete // was made. The txn is rolled back if it is just viewing the data. @@ -1639,26 +1641,20 @@ func BenchmarkBadger_Write(b *testing.B) { */ } -func reportTestBadgersNeedingClose() { +func reportTestBadgersNeedingClose() error { globalBadgerReg.mu.Lock() defer globalBadgerReg.mu.Unlock() n := len(globalBadgerReg.mp) - if n > 0 { - AlwaysPrintf("*** these badgers are still open (n=%v):", n) - i := 0 - for w := range globalBadgerReg.mp { - AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack) - i++ - } + if n == 0 { + return nil } -} - -var _ = reportTestBadgersNeedingClose // happy linter - -func TestMain(m *testing.M) { - ret := m.Run() - //reportTestBadgersNeedingClose() - os.Exit(ret) + AlwaysPrintf("*** these badgers are still open (n=%v):", n) + i := 0 + for w := range globalBadgerReg.mp { + AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack) + i++ + } + return errors.New("unclosed badgers, contact Animal Control") } /* diff --git a/blake3_test.go b/blake3_test.go index 86bbbdc75..f5111e9b3 100644 --- a/blake3_test.go +++ b/blake3_test.go @@ -21,6 +21,8 @@ import ( "testing" "encoding/hex" + + "github.com/pilosa/pilosa/v2/testhook" ) func TestBlake3Hasher(t *testing.T) { @@ -49,7 +51,7 @@ func TestCryptoRandInt64(t *testing.T) { } func TestHashOfDir(t *testing.T) { - dir, err := ioutil.TempDir(".", "TestHashOfDir-dir") + dir, err := testhook.TempDir(t, "TestHashOfDir-dir") panicOn(err) b := dir + sep + "A" + sep + "B" c := dir + sep + "A" + sep + "C" @@ -59,7 +61,6 @@ func TestHashOfDir(t *testing.T) { panicOn(ioutil.WriteFile(b+sep+"b_content", bmessage, 0644)) cmessage := []byte("hello C\n") panicOn(ioutil.WriteFile(c+sep+"c_content", cmessage, 0644)) - defer os.RemoveAll(dir) hsh := HashOfDir(dir) c2message := []byte("hello C2\n") diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 32ceec7f7..a296a8dda 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -17,7 +17,6 @@ package pilosa import ( "bytes" "fmt" - "io/ioutil" "math/rand" "net" "net/http" @@ -35,6 +34,7 @@ import ( "github.com/gorilla/mux" "github.com/pilosa/pilosa/v2/logger" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" ) @@ -91,14 +91,17 @@ func TestFragCombos(t *testing.T) { } // newIndexWithTempPath returns a new instance of Index. -func newIndexWithTempPath(name string) *Index { - path, err := ioutil.TempDir(*TempDir, "pilosa-index-") +func newIndexWithTempPath(tb testing.TB, name string) *Index { + path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-") if err != nil { panic(err) } h := NewHolder(DefaultPartitionN) h.Path = path index, err := h.CreateIndex(name, IndexOptions{}) + testhook.Cleanup(tb, func() { + h.Close() + }) if err != nil { panic(err) } @@ -158,7 +161,7 @@ func TestFragSources(t *testing.T) { c5.addNodeBasicSorted(node2) c5.addNodeBasicSorted(node3) - idx := newIndexWithTempPath("i") + idx := newIndexWithTempPath(t, "i") defer idx.Close() // Obtain transaction. @@ -398,7 +401,7 @@ func TestHasher(t *testing.T) { // Ensure ContainsShards can find the actual shard list for node and index. func TestCluster_ContainsShards(t *testing.T) { - c := NewTestCluster(5) + c := NewTestCluster(t, 5) c.ReplicaN = 3 shards := c.containsShards("test", roaring.NewBitmap(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10), c.nodes[2]) @@ -544,7 +547,7 @@ func TestCluster_Coordinator(t *testing.T) { } func TestCluster_Topology(t *testing.T) { - c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} + c1 := NewTestCluster(t, 1) // automatically creates Node{ID: "node0"} uri0 := NewTestURIFromHostPort("host0", 0) uri1 := NewTestURIFromHostPort("host1", 0) @@ -592,7 +595,7 @@ func TestCluster_Topology(t *testing.T) { func TestCluster_ResizeStates(t *testing.T) { t.Run("Single node, no data", func(t *testing.T) { - tc := NewClusterCluster(1) + tc := NewClusterCluster(t, 1) // Open TestCluster. if err := tc.Open(); err != nil { @@ -622,7 +625,7 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Single node, in topology", func(t *testing.T) { - tc := NewClusterCluster(0) + tc := NewClusterCluster(t, 0) if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } @@ -654,7 +657,7 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Single node, not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) + tc := NewClusterCluster(t, 0) if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } @@ -683,7 +686,7 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Multiple nodes, no data", func(t *testing.T) { - tc := NewClusterCluster(0) + tc := NewClusterCluster(t, 0) if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } @@ -725,7 +728,7 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { - tc := NewClusterCluster(0) + tc := NewClusterCluster(t, 0) if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } @@ -774,7 +777,7 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Multiple nodes, with data", func(t *testing.T) { - tc := NewClusterCluster(0) + tc := NewClusterCluster(t, 0) if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } @@ -927,7 +930,7 @@ func TestAE(t *testing.T) { // Ensures that coordinator can be changed. func TestCluster_UpdateCoordinator(t *testing.T) { t.Run("UpdateCoordinator", func(t *testing.T) { - c := NewTestCluster(2) + c := NewTestCluster(t, 2) oldNode := c.nodes[0] newNode := c.nodes[1] diff --git a/ctl/export_test.go b/ctl/export_test.go index 82c57816b..66314a4ef 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -46,7 +46,7 @@ func TestExportCommand_Validation(t *testing.T) { func TestExportCommand_Run(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) diff --git a/ctl/import_test.go b/ctl/import_test.go index 5c41a840c..ff5db6295 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -73,7 +73,7 @@ func TestImportCommand_Basic(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" @@ -102,7 +102,7 @@ func TestImportCommand_Basic(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) cm.Host = cmd.API.Node().URI.HostPort() cm.Index = "i" @@ -135,7 +135,7 @@ func TestImportCommand_RunValue(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) @@ -177,7 +177,7 @@ func TestImportCommand_RunValue(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) @@ -219,7 +219,7 @@ func TestImportCommand_RunKeys(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) @@ -271,8 +271,8 @@ func TestImportCommand_KeyReplication(t *testing.T) { c := test.MustRunCluster(t, 2) defer c.Close() - cmd0 := c[0] - cmd1 := c[1] + cmd0 := c.GetNode(0) + cmd1 := c.GetNode(1) host0 := cmd0.API.Node().URI.HostPort() host1 := cmd1.API.Node().URI.HostPort() @@ -339,7 +339,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) @@ -365,7 +365,7 @@ func TestImportCommand_RunValueKeys(t *testing.T) { func TestImportCommand_InvalidFile(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) @@ -453,7 +453,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { func TestImportCommand_BugOverwriteValue(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) @@ -529,7 +529,7 @@ func TestImportCommand_RunBool(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) cm.Host = cmd.API.Node().URI.HostPort() resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(""))) diff --git a/diagnostics_internal_test.go b/diagnostics_internal_test.go index 3966b2684..b2c7deedc 100644 --- a/diagnostics_internal_test.go +++ b/diagnostics_internal_test.go @@ -29,6 +29,7 @@ import ( func TestDiagnosticsClient(t *testing.T) { // Mock server. server := httptest.NewServer(nil) + defer server.Close() // Create a new client. d := newDiagnosticsCollector(server.URL) @@ -121,6 +122,7 @@ func TestDiagnosticsVersion_Check(t *testing.T) { t.Fatalf("couldn't encode version response: %v", err) } })) + defer server.Close() // Create a new client. d := newDiagnosticsCollector("localhost:10101") @@ -158,6 +160,7 @@ func compareJSON(a, b []byte) (bool, error) { func BenchmarkDiagnostics(b *testing.B) { // Mock server. server := httptest.NewServer(nil) + defer server.Close() // Create a new client. d := newDiagnosticsCollector(server.URL) diff --git a/executor.go b/executor.go index f329e336e..a0b0b75b4 100644 --- a/executor.go +++ b/executor.go @@ -29,6 +29,7 @@ import ( pb "github.com/pilosa/pilosa/v2/proto" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" ) @@ -100,6 +101,7 @@ func newExecutor(opts ...executorOption) *executor { // the few tests we've done at scale with concurrent query // workloads. Possible that it could be smaller. e.work = make(chan job, e.workerPoolSize) + _ = testhook.Opened(NewAuditor(), e, nil) for i := 0; i < e.workerPoolSize; i++ { e.workersWG.Add(1) go func() { @@ -114,6 +116,7 @@ func (e *executor) Close() error { e.workMu.Lock() defer e.workMu.Unlock() e.shutdown = true + _ = testhook.Closed(NewAuditor(), e, nil) close(e.work) e.workersWG.Wait() return nil diff --git a/executor_internal_test.go b/executor_internal_test.go index 8ae31da98..2c7ae1bd0 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -18,24 +18,25 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" "strconv" "strings" "testing" "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/testhook" ) func TestExecutor_TranslateGroupByCall(t *testing.T) { holder := NewHolder(DefaultPartitionN) + defer holder.Close() - cluster := NewTestCluster(1) + cluster := NewTestCluster(t, 1) e := &executor{ Holder: holder, Cluster: cluster, } - e.Holder.Path, _ = ioutil.TempDir(*TempDir, "") + e.Holder.Path, _ = testhook.TempDirInDir(t, *TempDir, "pilosa-executor-") err := e.Holder.Open() if err != nil { t.Fatalf("opening holder: %v", err) @@ -139,9 +140,9 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) { e := &executor{ Holder: holder, - Cluster: NewTestCluster(1), + Cluster: NewTestCluster(t, 1), } - e.Holder.Path, _ = ioutil.TempDir(*TempDir, "") + e.Holder.Path, _ = testhook.TempDirInDir(t, *TempDir, "pilosa-executor-") if err := e.Holder.Open(); err != nil { t.Fatalf("opening holder: %v", err) } diff --git a/executor_test.go b/executor_test.go index e29b8d93b..3abaebf46 100644 --- a/executor_test.go +++ b/executor_test.go @@ -39,6 +39,7 @@ import ( "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" ) @@ -164,7 +165,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 1) hldr.SetBit("i", "general", 10, 2) @@ -172,7 +173,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, 4) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference(Row(general=10), Row(general=11))`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { t.Fatalf("unexpected columns: %+v", columns) @@ -230,10 +231,10 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Empty_Difference(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 1) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference()`}); err == nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Difference()`}); err == nil { t.Fatalf("Empty Difference query should give error, but got %v", res) } } @@ -243,7 +244,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 1) hldr.SetBit("i", "general", 10, ShardWidth+1) hldr.SetBit("i", "general", 10, ShardWidth+2) @@ -251,7 +252,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect(Row(general=10), Row(general=11))`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) @@ -313,7 +314,7 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect()`}); err == nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Intersect()`}); err == nil { t.Fatalf("Empty Intersect query should give error, but got %v", res) } } @@ -323,7 +324,7 @@ func TestExecutor_Execute_Union(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 0) hldr.SetBit("i", "general", 10, ShardWidth+1) hldr.SetBit("i", "general", 10, ShardWidth+2) @@ -331,7 +332,7 @@ func TestExecutor_Execute_Union(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union(Row(general=10), Row(general=11))`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { t.Fatalf("unexpected columns: %+v", columns) @@ -392,10 +393,10 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Empty_Union(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 0) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union()`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Union()`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected columns: %+v", columns) @@ -407,7 +408,7 @@ func TestExecutor_Execute_Xor(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 0) hldr.SetBit("i", "general", 10, ShardWidth+1) @@ -416,7 +417,7 @@ func TestExecutor_Execute_Xor(t *testing.T) { hldr.SetBit("i", "general", 11, 2) hldr.SetBit("i", "general", 11, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Xor(Row(general=10), Row(general=11))`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Xor(Row(general=10), Row(general=11))`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) @@ -478,13 +479,13 @@ func TestExecutor_Execute_Count(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "f", 10, 3) hldr.SetBit("i", "f", 10, ShardWidth+1) hldr.SetBit("i", "f", 10, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { t.Fatal(err) } else if res.Results[0] != uint64(3) { t.Fatalf("unexpected n: %d", res.Results[0]) @@ -544,9 +545,8 @@ func TestExecutor_Execute_Set(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] - holder := cmd.Server.Holder() - hldr := test.Holder{Holder: holder} + cmd := cluster.GetNode(0) + hldr := cluster.GetHolder(0) hldr.SetBit("i", "f", 1, 0) // creates and commits a Tx internally. t.Run("OK", func(t *testing.T) { @@ -605,9 +605,8 @@ func TestExecutor_Execute_Set(t *testing.T) { t.Run("RowKeyColumnKey", func(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] - holder := cmd.Server.Holder() - hldr := test.Holder{Holder: holder} + cmd := cluster.GetNode(0) + hldr := cluster.GetHolder(0) idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) t.Run("OK", func(t *testing.T) { @@ -774,7 +773,7 @@ func TestExecutor_Execute_SetBool(t *testing.T) { t.Run("Basic", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -783,35 +782,35 @@ func TestExecutor_Execute_SetBool(t *testing.T) { } // Set a true bit. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { t.Fatal(err) } else if !res.Results[0].(bool) { t.Fatalf("expected column changed") } // Set the same bit to true again verify nothing changed. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=true)`}); err != nil { t.Fatal(err) } else if res.Results[0].(bool) { t.Fatalf("expected column to be unchanged") } // Set the same bit to false. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=false)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=false)`}); err != nil { t.Fatal(err) } else if !res.Results[0].(bool) { t.Fatalf("expected column changed") } // Ensure that the false row is set. - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=false)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=false)`}); err != nil { t.Fatal(err) } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{100}) { t.Fatalf("unexpected colums: %+v", columns) } // Ensure that the true row is empty. - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=true)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=true)`}); err != nil { t.Fatal(err) } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { t.Fatalf("unexpected colums: %+v", columns) @@ -820,7 +819,7 @@ func TestExecutor_Execute_SetBool(t *testing.T) { t.Run("Error", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -829,12 +828,12 @@ func TestExecutor_Execute_SetBool(t *testing.T) { } // Set bool using a string value. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f="true")`}); err == nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f="true")`}); err == nil { t.Fatalf("expected invalid bool type error") } // Set bool using an integer. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=1)`}); err == nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=1)`}); err == nil { t.Fatalf("expected invalid bool type error") } @@ -846,7 +845,7 @@ func TestExecutor_Execute_SetDecimal(t *testing.T) { t.Run("Basic", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -855,26 +854,26 @@ func TestExecutor_Execute_SetDecimal(t *testing.T) { } // Set a value. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1000, f=1.5)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1000, f=1.5)`}); err != nil { t.Fatal(err) } else if !res.Results[0].(bool) { t.Fatalf("expected column changed") } // Set the same value again verify nothing changed. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1000, f=1.5)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1000, f=1.5)`}); err != nil { t.Fatal(err) } else if res.Results[0].(bool) { t.Fatalf("expected column to be unchanged") } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f == 1.5)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f == 1.5)`}); err != nil { t.Fatal(err) } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1000}) { t.Fatalf("unexpected colums: %+v", columns) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f > 1.4999)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f > 1.4999)`}); err != nil { t.Fatal(err) } else if columns := result.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1000}) { t.Fatalf("unexpected colums: %+v", columns) @@ -883,7 +882,7 @@ func TestExecutor_Execute_SetDecimal(t *testing.T) { t.Run("Error", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -892,7 +891,7 @@ func TestExecutor_Execute_SetDecimal(t *testing.T) { } // Set decimal using a string value. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1000, f="1.5")`}); err == nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1000, f="1.5")`}); err == nil { t.Fatalf("expected invalid decimal type error") } }) @@ -902,12 +901,12 @@ func TestExecutor_Execute_SetDecimal(t *testing.T) { func TestExecutor_Execute_OldPQL(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // set a bit so the view gets created. hldr.SetBit("i", "f", 1, 0) - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetBit(frame=f, row=11, col=1)`}); err == nil || errors.Cause(err).Error() != "unknown call: SetBit" { t.Fatalf("Expected error: 'unknown call: SetBit', got: %v. Full: %v", errors.Cause(err), err) } } @@ -917,7 +916,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -928,9 +927,9 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } // Set bsiGroup values. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f=25)`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f=25)`}); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=10)`}); err != nil { + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(100, f=10)`}); err != nil { t.Fatal(err) } @@ -962,7 +961,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { t.Run("", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { @@ -970,19 +969,19 @@ func TestExecutor_Execute_SetValue(t *testing.T) { } t.Run("ErrColumnBSIGroupRequired", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `Set() column argument 'col' required` { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(invalid_column_name=10, f=100)`}); err == nil || errors.Cause(err).Error() != `Set() column argument 'col' required` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrColumnBSIGroupValue", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set("bad_column", f=100)`}); err == nil || errors.Cause(err).Error() != `string 'col' value not allowed unless index 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) t.Run("ErrInvalidBSIGroupValueType", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f="hello")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(10, f="hello")`}); err == nil || errors.Cause(err).Error() != `string 'row' value not allowed unless field 'keys' option enabled` { t.Fatalf("unexpected error: %s", err) } }) @@ -993,7 +992,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { func TestExecutor_Execute_SetRowAttrs(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Create fields. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) @@ -1007,16 +1006,16 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { t.Run("rowID", func(t *testing.T) { // Set two attrs on f/10. // Also set attrs on other rows and fields to test isolation. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 200, YYY=1)`}); err != nil { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(xxx, 10, YYY=1)`}); err != nil { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(f, 10, baz=123, bat=true)`}); err != nil { t.Fatal(err) } @@ -1031,17 +1030,17 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) { t.Run("rowKey", func(t *testing.T) { // Set two attrs on f/10. // Also set attrs on other rows and fields to test isolation. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", foo="bar")`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", foo="bar")`}); err != nil { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row200", YYY=1)`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row200", YYY=1)`}); err != nil { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", baz=123, bat=true)`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `SetRowAttrs(kf, "row10", baz=123, bat=true)`}); err != nil { t.Fatal(err) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(kf="row10")`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(kf="row10")`}); err != nil { t.Fatal(err) } else if attrs := result.Results[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { t.Fatalf("unexpected attrs: %+v", attrs) @@ -1054,7 +1053,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { @@ -1063,7 +1062,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) Set(1, f=0) Set(` + strconv.Itoa(ShardWidth) + `, f=0) @@ -1077,12 +1076,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } - err := c[0].RecalculateCaches(t) + err := c.GetNode(0).RecalculateCaches(t) if err != nil { t.Fatalf("recalculating caches: %v", err) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{ Pairs: []pilosa.Pair{ @@ -1098,7 +1097,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Run("RowIDColumnKey", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { @@ -1107,7 +1106,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("other"); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set("zero", f=0) Set("one", f=0) Set("sw", f=0) @@ -1121,12 +1120,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } - err := c[0].RecalculateCaches(t) + err := c.GetNode(0).RecalculateCaches(t) if err != nil { t.Fatalf("recalculating caches: %v", err) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{ Pairs: []pilosa.Pair{ @@ -1142,7 +1141,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Run("RowKeyColumnKey", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { @@ -1151,7 +1150,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("other", pilosa.OptFieldKeys()); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set("zero", f="zero") Set("one", f="zero") Set("sw", f="zero") @@ -1165,12 +1164,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } - err := c[0].RecalculateCaches(t) + err := c.GetNode(0).RecalculateCaches(t) if err != nil { t.Fatalf("recalculating caches: %v", err) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) } else { if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{ @@ -1188,7 +1187,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Run("RowKeyColumnKey", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Set columns for rows 0, 10, & 20 across two shards. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}); err != nil { @@ -1197,7 +1196,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("other", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set("a", f="foo") Set("b", f="foo") Set("c", f="foo") @@ -1211,12 +1210,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } - err := c[0].RecalculateCaches(t) + err := c.GetNode(0).RecalculateCaches(t) if err != nil { t.Fatalf("recalculating caches: %v", err) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) } else if diff := cmp.Diff(result.Results, []interface{}{ &pilosa.PairsField{ @@ -1234,24 +1233,24 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Run("ErrFieldNotFound", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Set data on the "f" field. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) Set(0, f=1) `}); err != nil { t.Fatal(err) - } else if err := c[0].RecalculateCaches(t); err != nil { + } else if err := c.GetNode(0).RecalculateCaches(t); err != nil { t.Fatalf("recalculating caches: %v", err) } // Attempt to query the "g" field. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(g, n=2)`}); err == nil || err.Error() != `executing: field "g" not found` { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(g, n=2)`}); err == nil || err.Error() != `executing: field "g" not found` { t.Fatalf("unexpected error: %v", err) } }) @@ -1259,14 +1258,14 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Run("ErrBSIField", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Create BSI "f" field. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 100)); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN() on integer field: "f"` { + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN() on integer field: "f"` { t.Fatalf("unexpected error: %v", err) } }) @@ -1274,18 +1273,18 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Run("ErrCacheNone", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeSet(pilosa.CacheTypeNone, 0)); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) Set(0, f=1) `}); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN(), field has no cache: "f"` { + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN(), field has no cache: "f"` { t.Fatalf("unexpected error: %v", err) } }) @@ -1294,7 +1293,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { func TestExecutor_Execute_TopN_fill(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) @@ -1305,7 +1304,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr.SetBit("i", "f", 1, ShardWidth) // Execute query. - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ Pairs: []pilosa.Pair{ @@ -1321,7 +1320,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { func TestExecutor_Execute_TopN_fill_small(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, ShardWidth) @@ -1342,7 +1341,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { hldr.SetBit("i", "f", 4, 3*ShardWidth+1) // Execute query. - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ Pairs: []pilosa.Pair{ @@ -1358,7 +1357,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { func TestExecutor_Execute_TopN_Src(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) // Set columns for rows 0, 10, & 20 across two shards. hldr.SetBit("i", "f", 0, 0) @@ -1375,13 +1374,13 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr.SetBit("i", "other", 100, ShardWidth+1) hldr.SetBit("i", "other", 100, ShardWidth+2) - err := c[0].RecalculateCaches(t) + err := c.GetNode(0).RecalculateCaches(t) if err != nil { t.Fatalf("recalculating caches: %v", err) } // Execute query. - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ Pairs: []pilosa.Pair{ @@ -1399,7 +1398,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { func TestExecutor_Execute_TopN_Attr(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) hldr.SetBit("i", "f", 10, ShardWidth) @@ -1407,7 +1406,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1, attrName="category", attrValues=[123])`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ Pairs: []pilosa.Pair{ @@ -1424,7 +1423,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "f", 0, 0) hldr.SetBit("i", "f", 0, 1) @@ -1433,7 +1432,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { if err := hldr.Field("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ Pairs: []pilosa.Pair{ @@ -1451,7 +1450,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Run("Int", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -1475,7 +1474,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(` Set(10, %s=%d) `, fld, test.set)}); err != nil { t.Fatal(err) @@ -1485,7 +1484,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Run("Min", func(t *testing.T) { pql = fmt.Sprintf(`Min(field=%s)`, fld) - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: test.set, Count: 1}) { t.Fatalf("unexpected min result, test %d: %s", i, spew.Sdump(result)) @@ -1494,7 +1493,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Run("Max", func(t *testing.T) { pql = fmt.Sprintf(`Max(field=%s)`, fld) - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: test.set, Count: 1}) { t.Fatalf("unexpected max result, test %d: %s", i, spew.Sdump(result)) @@ -1507,7 +1506,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Run("Decimal", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -1556,28 +1555,28 @@ func TestExecutor_Execute_MinMax(t *testing.T) { if _, err := idx.CreateFieldIfNotExists("z", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, z=0)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, z=0)`}); err != nil { t.Fatal(err) } else if !res.Results[0].(bool) { t.Fatalf("expected column changed") } // set things in other shards, that won't have decimal values - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1234567, z=0)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1234567, z=0)`}); err != nil { t.Fatal(err) } else if !res.Results[0].(bool) { t.Fatalf("expected column changed") } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2345678, z=0)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2345678, z=0)`}); err != nil { t.Fatal(err) } else if !res.Results[0].(bool) { t.Fatalf("expected column changed") } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(3456789, z=0)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(3456789, z=0)`}); err != nil { t.Fatal(err) } else if !res.Results[0].(bool) { t.Fatalf("expected column changed") } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(4567890, z=0)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(4567890, z=0)`}); err != nil { t.Fatal(err) } else if !res.Results[0].(bool) { t.Fatalf("expected column changed") @@ -1589,7 +1588,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(` Set(6700000, %s=%s) `, fld, test.set)}); err != nil { t.Fatal(err) @@ -1599,7 +1598,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Run("Min", func(t *testing.T) { pql = fmt.Sprintf(`Min(field=%s)`, fld) - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{DecimalVal: &test.exp, Count: 1}) { t.Fatalf("unexpected min result, test %d: %s", i, spew.Sdump(result)) @@ -1608,7 +1607,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Run("Max", func(t *testing.T) { pql = fmt.Sprintf(`Max(field=%s)`, fld) - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{DecimalVal: &test.exp, Count: 1}) { t.Fatalf("unexpected max result, test %d: %s", i, spew.Sdump(result)) @@ -1622,7 +1621,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Run("ColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -1637,7 +1636,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, x=0) Set(3, x=0) Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) @@ -1674,7 +1673,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } else { pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) @@ -1686,7 +1685,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Run("ColumnKey", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}) if err != nil { @@ -1701,7 +1700,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set("zero", x=0) Set("three", x=0) Set("sw1", x=0) @@ -1738,7 +1737,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } else { pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) @@ -1764,7 +1763,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } else { pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) @@ -1779,7 +1778,7 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { t.Run("RowID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -1790,7 +1789,7 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=7000) Set(3, f=50) Set(` + strconv.Itoa(ShardWidth+1) + `, f=10000) @@ -1801,7 +1800,7 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { } t.Run("MinRow", func(t *testing.T) { - result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "MinRow(field=f)"}) + result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "MinRow(field=f)"}) if err != nil { t.Fatal(err) } @@ -1815,7 +1814,7 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { }) t.Run("MaxRow", func(t *testing.T) { - result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "MaxRow(field=f)"}) + result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "MaxRow(field=f)"}) if err != nil { t.Fatal(err) } @@ -1832,7 +1831,7 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { t.Run("RowKey", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -1843,7 +1842,7 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f="seven-thousand") Set(3, f="fifty") Set(` + strconv.Itoa(ShardWidth+1) + `, f="ten-thousand") @@ -1854,7 +1853,7 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { } t.Run("MinRow", func(t *testing.T) { - result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "MinRow(field=f)"}) + result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "MinRow(field=f)"}) if err != nil { t.Fatal(err) } @@ -1868,7 +1867,7 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { }) t.Run("MaxRow", func(t *testing.T) { - result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "MaxRow(field=f)"}) + result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "MaxRow(field=f)"}) if err != nil { t.Fatal(err) } @@ -1888,7 +1887,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("ColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -1915,7 +1914,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, x=0) Set(` + strconv.Itoa(ShardWidth+1) + `, x=0) @@ -1936,7 +1935,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("Integer", func(t *testing.T) { t.Run("NoFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -1944,7 +1943,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -1954,7 +1953,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("Decimal", func(t *testing.T) { t.Run("NoFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=dec)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=dec)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{DecimalVal: &pql.Decimal{Value: 700007, Scale: 3}, Count: 3}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -1962,7 +1961,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=dec)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=dec)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{DecimalVal: &pql.Decimal{Value: 500005, Scale: 3}, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -1974,7 +1973,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("ColumnKey", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{Keys: true}) if err != nil { @@ -1997,7 +1996,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set("zero", x=0) Set("sw1", x=0) @@ -2013,7 +2012,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { } t.Run("NoFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(field=foo)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2021,7 +2020,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Sum(Row(x=0), field=foo)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2376,7 +2375,7 @@ func TestExecutor_Execute_Range_Deprecated(t *testing.T) { func TestExecutor_DecimalArgs(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -2396,7 +2395,7 @@ func TestExecutor_DecimalArgs(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) `}); err != nil { t.Fatal(err) @@ -2407,7 +2406,7 @@ func TestExecutor_DecimalArgs(t *testing.T) { func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{TrackExistence: true}) if err != nil { @@ -2434,7 +2433,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) @@ -2453,7 +2452,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Run("EQ", func(t *testing.T) { // EQ null - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other == null)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other == null)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{1, 50, @@ -2465,7 +2464,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } // EQ - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil { t.Fatal(err) } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, (5 * ShardWidth) + 100}; !reflect.DeepEqual(exp, got) { t.Fatalf("Query().Row.Columns=%#v, expected %#v", got, exp) @@ -2474,19 +2473,19 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { t.Run("NEQ", func(t *testing.T) { // NEQ null - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != null)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != null)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } // NEQ - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo != 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo != 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %#v", result.Results[0].(*pilosa.Row).Columns()) } // NEQ - - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != -20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != -20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2495,7 +2494,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo < 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo < 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2503,7 +2502,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("LTE", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo <= 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo <= 20)`}); err != nil { t.Fatal(err) } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}; !reflect.DeepEqual(got, exp) { t.Fatalf("unexpected result: got=%v, exp=%v", got, exp) @@ -2511,7 +2510,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo > 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo > 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) @@ -2519,7 +2518,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("GTE", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo >= 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo >= 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) @@ -2552,7 +2551,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { if test.exp { expected = []uint64{0} } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(expected, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result for query: %s (%#v)", test.q, result.Results[0].(*pilosa.Row).Columns()) @@ -2564,7 +2563,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { // Ensure that the NotNull code path gets run. t.Run("NotNull", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(0 <= other <= 1000)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(0 <= other <= 1000)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2572,7 +2571,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("BelowMin", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 0)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 0)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2580,7 +2579,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("AboveMax", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 200)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 200)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2588,7 +2587,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("LTAboveMax", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge < 200)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge < 200)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) @@ -2596,7 +2595,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge > -1000)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge > -1000)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) @@ -2604,7 +2603,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("ErrFieldNotFound", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound { t.Fatal(err) } }) @@ -2614,7 +2613,7 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { func TestExecutor_Execute_Row_BSIGroupEdge(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -2629,13 +2628,13 @@ func TestExecutor_Execute_Row_BSIGroupEdge(t *testing.T) { // Set a value at the edge of bitDepth (i.e. 2^n-1; here, n=3). // It must also be the max value in the field; in other words, // set the value to bsiGroup.bitDepthMax(). - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(100, f1=7) `}); err != nil { t.Fatal(err) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f1 < 10)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f1 < 10)`}); err != nil { t.Fatal(err) } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{100}; !reflect.DeepEqual(got, exp) { t.Fatalf("unexpected result: got=%v, exp=%v", got, exp) @@ -2650,13 +2649,13 @@ func TestExecutor_Execute_Row_BSIGroupEdge(t *testing.T) { // Set a value at the negative edge of bitDepth (i.e. -(2^n-1); here, n=3). // It must also be the min value in the field; in other words, // set the value to bsiGroup.bitDepthMin(). - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(200, f2=-7) `}); err != nil { t.Fatal(err) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f2 > -10)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f2 > -10)`}); err != nil { t.Fatal(err) } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{200}; !reflect.DeepEqual(got, exp) { t.Fatalf("unexpected result: got=%v, exp=%v", got, exp) @@ -2669,7 +2668,7 @@ func TestExecutor_Execute_Row_BSIGroupEdge(t *testing.T) { } // Set a value anywhere in range. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(300, f3=10) `}); err != nil { t.Fatal(err) @@ -2688,7 +2687,7 @@ func TestExecutor_Execute_Row_BSIGroupEdge(t *testing.T) { for i, test := range tests { pql := fmt.Sprintf("Row(%d < f3 < %d)", test.predA, test.predB) - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if got, exp := result.Results[0].(*pilosa.Row).Columns(), []uint64{}; !reflect.DeepEqual(got, exp) { t.Fatalf("test %d unexpected result: got=%v, exp=%v", i, got, exp) @@ -2701,7 +2700,7 @@ func TestExecutor_Execute_Row_BSIGroupEdge(t *testing.T) { func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -2728,7 +2727,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=0) Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) @@ -2746,7 +2745,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { } t.Run("EQ", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2755,19 +2754,19 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { t.Run("NEQ", func(t *testing.T) { // NEQ null - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != null)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != null)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo != 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo != 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != -20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(other != -20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2776,7 +2775,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo < 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo < 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2784,7 +2783,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("LTE", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo <= 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo <= 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2792,7 +2791,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo > 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo > 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2800,7 +2799,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("GTE", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo >= 20)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo >= 20)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2808,7 +2807,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 < other < 1000)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 < other < 1000)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2817,7 +2816,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { // Ensure that the NotNull code path gets run. t.Run("NotNull", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 <= other <= 1000)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 <= other <= 1000)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2825,7 +2824,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("BelowMin", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 0)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 0)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2833,7 +2832,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("AboveMax", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 200)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(foo == 200)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2841,7 +2840,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("LTAboveMax", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge < 200)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge < 200)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) @@ -2849,7 +2848,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("GTBelowMin", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge > -1200)`}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(edge > -1200)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) @@ -2857,7 +2856,7 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { }) t.Run("ErrFieldNotFound", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound { t.Fatal(err) } }) @@ -2872,14 +2871,14 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { server.OptCommandServerOptions(pilosa.OptServerNodeID("node1"), pilosa.OptServerClusterHasher(&test.ModHasher{}))}, ) defer c.Close() - hldr0 := test.Holder{Holder: c[0].Server.Holder()} - hldr1 := test.Holder{Holder: c[1].Server.Holder()} + hldr0 := c.GetHolder(0) + hldr1 := c.GetHolder(1) - _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + _, err := c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) if err != nil { t.Fatalf("creating field: %v", err) } @@ -2887,14 +2886,14 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { hldr1.MustSetBits("i", "f", 10, ShardWidth+1, ShardWidth+2, (3*ShardWidth)+4) hldr0.SetBit("i", "f", 10, 1) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, ShardWidth + 1, ShardWidth + 2, (3 * ShardWidth) + 4}) { t.Fatalf("unexpected columns: %+v", columns) } t.Run("Count", func(t *testing.T) { - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Count(Row(f=10))`}); err != nil { t.Fatal(err) } else if res.Results[0] != uint64(4) { t.Fatalf("unexpected n: %d", res.Results[0]) @@ -2902,7 +2901,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("Remote SetBit", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, f=7)`, pilosa.ShardWidth+1)}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, f=7)`, pilosa.ShardWidth+1)}); err != nil { t.Fatalf("querying remote: %v", err) } @@ -2912,12 +2911,12 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("remote with timestamp", func(t *testing.T) { - _, err = c[0].API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y")) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y")) if err != nil { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, z=5, 2010-07-08T00:00)`, pilosa.ShardWidth+1)}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, z=5, 2010-07-08T00:00)`, pilosa.ShardWidth+1)}); err != nil { t.Fatalf("quuerying remote: %v", err) } @@ -2927,11 +2926,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("remote topn", func(t *testing.T) { - _, err = c[0].API.CreateField(context.Background(), "i", "fn", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "fn", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) if err != nil { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(500001, fn=5) Set(1500001, fn=5) Set(2500001, fn=5) @@ -2944,12 +2943,12 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { `}); err != nil { t.Fatalf("querying remote: %v", err) } - err := c[0].API.RecalculateCaches(context.Background()) + err := c.GetNode(0).API.RecalculateCaches(context.Background()) if err != nil { t.Fatalf("recalculating caches: %v", err) } - if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + if res, err := c.GetNode(1).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: `TopN(fn, n=3)`, }); err != nil { @@ -2967,7 +2966,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("remote setrowattrs", func(t *testing.T) { - if _, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + if _, err := c.GetNode(1).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: `SetRowAttrs(_field="f", _row=10, bat=true, baz=123)`, }); err != nil { @@ -2978,7 +2977,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("remote groupBy", func(t *testing.T) { - if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + if res, err := c.GetNode(1).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: `GroupBy(Rows(f))`, }); err != nil { @@ -2994,11 +2993,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("remote groupBy on ints", func(t *testing.T) { - _, err = c[0].API.CreateField(context.Background(), "i", "fint", pilosa.OptFieldTypeInt(-1000, 1000)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "fint", pilosa.OptFieldTypeInt(-1000, 1000)) if err != nil { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, fint=1) Set(1, fint=2) @@ -3016,7 +3015,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("querying remote: %v", err) } - if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + if res, err := c.GetNode(1).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: `GroupBy(Rows(fint), limit=4, filter=Union(Row(fint < 1), Row(fint > 2)))`, }); err != nil { @@ -3036,11 +3035,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("groupBy on ints with offset regression", func(t *testing.T) { - _, err = c[0].API.CreateField(context.Background(), "i", "hint", pilosa.OptFieldTypeInt(1, 1000)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "hint", pilosa.OptFieldTypeInt(1, 1000)) if err != nil { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, hint=1) Set(1, hint=2) Set(2, hint=3) @@ -3048,7 +3047,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("querying remote: %v", err) } - if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + if res, err := c.GetNode(1).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: `GroupBy(Rows(hint))`, }); err != nil { @@ -3067,16 +3066,16 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("Row on ints with ASSIGN condition", func(t *testing.T) { - _, err := c[0].API.CreateIndex(context.Background(), "intidx", pilosa.IndexOptions{}) + _, err := c.GetNode(0).API.CreateIndex(context.Background(), "intidx", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "intidx", "gint", pilosa.OptFieldTypeInt(-1000, 1000)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "intidx", "gint", pilosa.OptFieldTypeInt(-1000, 1000)) if err != nil { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "intidx", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "intidx", Query: ` Set(1000, gint=1) Set(2000, gint=2) Set(3000, gint=3) @@ -3084,7 +3083,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("querying remote: %v", err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "intidx", Query: `Row(gint=2)Row(gint==1)`, }); err != nil { @@ -3102,16 +3101,16 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("Row on decimals with ASSIGN condition", func(t *testing.T) { - _, err := c[0].API.CreateIndex(context.Background(), "decidx", pilosa.IndexOptions{}) + _, err := c.GetNode(0).API.CreateIndex(context.Background(), "decidx", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "decidx", "fdec", pilosa.OptFieldTypeDecimal(0)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "decidx", "fdec", pilosa.OptFieldTypeDecimal(0)) if err != nil { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "decidx", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "decidx", Query: ` Set(11, fdec=1.1) Set(22, fdec=2.2) Set(33, fdec=3.3) @@ -3119,7 +3118,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("querying remote: %v", err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "decidx", Query: `Row(fdec=2.2)Row(fdec==1.1)`, }); err != nil { @@ -3136,19 +3135,19 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("Row on foreign key with ASSIGN condition", func(t *testing.T) { - _, err := c[0].API.CreateIndex(context.Background(), "parent", pilosa.IndexOptions{Keys: true}) + _, err := c.GetNode(0).API.CreateIndex(context.Background(), "parent", pilosa.IndexOptions{Keys: true}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "parent", "general", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "parent", "general", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) if err != nil { t.Fatalf("creating field: %v", err) } - _, err = c[0].API.CreateIndex(context.Background(), "child", pilosa.IndexOptions{Keys: false}) + _, err = c.GetNode(0).API.CreateIndex(context.Background(), "child", pilosa.IndexOptions{Keys: false}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "child", "parentid", + _, err = c.GetNode(0).API.CreateField(context.Background(), "child", "parentid", pilosa.OptFieldForeignIndex("parent"), pilosa.OptFieldTypeInt(-9223372036854775808, 9223372036854775807), ) @@ -3156,7 +3155,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "child", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "child", Query: ` Set(1, parentid="one") Set(2, parentid="two") Set(3, parentid="three") @@ -3164,7 +3163,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("querying remote: %v", err) } - if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + if res, err := c.GetNode(1).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "child", Query: `Row(parentid="two")Row(parentid=="one")`, }); err != nil { @@ -3185,14 +3184,15 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { // Ensure executor returns an error if too many writes are in a single request. func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { c := test.MustNewCluster(t, 1) - c[0].Config.MaxWritesPerRequest = 3 + defer c.Close() + c.GetNode(0).Config.MaxWritesPerRequest = 3 err := c.Start() if err != nil { t.Fatal(err) } - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set() Clear() Set() Set()`}); errors.Cause(err) != pilosa.ErrTooManyWrites { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set() Clear() Set() Set()`}); errors.Cause(err) != pilosa.ErrTooManyWrites { t.Fatalf("unexpected error: %s", err) } } @@ -3201,7 +3201,7 @@ func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) { func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) @@ -3213,11 +3213,11 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { } // SetColumnAttrs call should exclude the field attribute - _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(10, f=1)"}) + _, err = c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(10, f=1)"}) if err != nil { t.Fatal(err) } - _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(10, foo='bar')"}) + _, err = c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(10, foo='bar')"}) if err != nil { t.Fatal(err) } @@ -3230,11 +3230,11 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { } // SetColumnAttrs call should not break if field is not specified - _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(20, f=10)"}) + _, err = c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(20, f=10)"}) if err != nil { t.Fatal(err) } - _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(20, foo='bar')"}) + _, err = c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "SetColumnAttrs(20, foo='bar')"}) if err != nil { t.Fatal(err) } @@ -3251,7 +3251,7 @@ func TestExecutor_SetColumnAttrs_ExcludeField(t *testing.T) { func TestExecutor_Time_Clear_Quantums(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) var rangeTests = []struct { quantum pilosa.TimeQuantum @@ -3292,13 +3292,13 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { t.Fatal(err) } // Populate - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: populateBatch}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: populateBatch}); err != nil { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: clearColumn}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: clearColumn}); err != nil { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: rangeCheckQuery}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: indexName, Query: rangeCheckQuery}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, tt.expected) { t.Fatalf("unexpected columns: %+v", columns) @@ -3439,7 +3439,7 @@ func TestExecutor_Execute_Existence(t *testing.T) { ), }) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) @@ -3448,7 +3448,7 @@ func TestExecutor_Execute_Existence(t *testing.T) { } // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+2, 20), @@ -3456,24 +3456,24 @@ func TestExecutor_Execute_Existence(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) { t.Fatalf("unexpected columns after Not: %+v", bits) } // Reopen cluster to ensure existence field is reloaded. - if err := c[0].Reopen(); err != nil { + if err := c.GetNode(0).Reopen(); err != nil { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) { t.Fatalf("unexpected columns after reopen: %+v", bits) @@ -3562,8 +3562,8 @@ func TestExecutor_Execute_FieldValue(t *testing.T) { c := test.MustRunCluster(t, 2) defer c.Close() - node0 := c[0] - node1 := c[1] + node0 := c.GetNode(0) + node1 := c.GetNode(1) // Index with IDs c.CreateField(t, "i", pilosa.IndexOptions{Keys: false}, "f", pilosa.OptFieldTypeInt(-1100, 1000)) @@ -3743,7 +3743,7 @@ func TestExecutor_Execute_All(t *testing.T) { t.Run("ColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) fld, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { @@ -3777,7 +3777,7 @@ func TestExecutor_Execute_All(t *testing.T) { req.RowIDs[bitCount-1] = 10 req.ColumnIDs[bitCount-1] = uint64((3 * ShardWidth) + 2) - if err := c[0].API.Import(context.Background(), req); err != nil { + if err := c.GetNode(0).API.Import(context.Background(), req); err != nil { t.Fatal(err) } @@ -3802,7 +3802,7 @@ func TestExecutor_Execute_All(t *testing.T) { {qry: fmt.Sprintf("All(limit=%d, offset=2)", bitCount-3), expCols: req.ColumnIDs[2 : bitCount-1], expCnt: uint64(bitCount - 3)}, } for i, test := range tests { - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { t.Fatal(err) } else if cnt := res.Results[0].(*pilosa.Row).Count(); cnt != test.expCnt { t.Fatalf("test %d, unexpected count, got: %d, but expected: %d", i, cnt, test.expCnt) @@ -3825,7 +3825,7 @@ func TestExecutor_Execute_All(t *testing.T) { ), }) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true, Keys: true}) fld, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { @@ -3851,7 +3851,7 @@ func TestExecutor_Execute_All(t *testing.T) { req.ColumnKeys[i] = fmt.Sprintf("c%d", i) } - if err := c[0].API.Import(context.Background(), req); err != nil { + if err := c.GetNode(0).API.Import(context.Background(), req); err != nil { t.Fatal(err) } @@ -3869,7 +3869,7 @@ func TestExecutor_Execute_All(t *testing.T) { {qry: "All(limit=4, offset=5)", expCols: nil, expCnt: 0}, } for i, test := range tests { - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { t.Fatal(err) } else if cnt := len(res.Results[0].(*pilosa.Row).Keys); uint64(cnt) != test.expCnt { t.Fatalf("test %d, unexpected count, got: %d, but expected: %d", i, cnt, test.expCnt) @@ -3888,13 +3888,13 @@ func TestExecutor_Execute_All(t *testing.T) { t.Run("AllShard", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(3001, f=3) Set(5001, f=5) Set(5002, f=5) @@ -3903,7 +3903,7 @@ Set(5002, f=5) } expCols := []uint64{5001, 5002} - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Intersect(All(), Row(f=5))"}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Intersect(All(), Row(f=5))"}); err != nil { t.Fatal(err) } else if cols := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(cols, expCols) { t.Fatalf("unexpected columns, got: %v, but expected: %v", cols, expCols) @@ -4036,7 +4036,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { t.Run("Int", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) _, err := index.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { @@ -4044,7 +4044,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { } // Ensure that clearing a row raises an error. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err == nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=1)`}); err == nil { t.Fatal("expected clear row to return an error") } }) @@ -4052,7 +4052,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { t.Run("TopN", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { @@ -4083,16 +4083,16 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { ` // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: cc}); err != nil { t.Fatal(err) } - if err := c[0].RecalculateCaches(t); err != nil { + if err := c.GetNode(0).RecalculateCaches(t); err != nil { t.Fatalf("recalculating caches: %v", err) } // Check the TopN results. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{ Pairs: []pilosa.Pair{ @@ -4106,14 +4106,14 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { } // Clear the row and ensure we get a `true` response. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=2)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `ClearRow(f=2)`}); err != nil { t.Fatal(err) } else if res := res.Results[0].(bool); !res { t.Fatalf("unexpected clear row result: %+v", res) } // Ensure that the cleared row doesn't show up in TopN (i.e. it was removed from the cache). - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{ Pairs: []pilosa.Pair{ @@ -4145,7 +4145,7 @@ func TestExecutor_Execute_SetRow(t *testing.T) { t.Run("Set_NewRow", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { t.Fatal(err) @@ -4155,7 +4155,7 @@ func TestExecutor_Execute_SetRow(t *testing.T) { } // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10), @@ -4163,35 +4163,35 @@ func TestExecutor_Execute_SetRow(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } // Store row 10 into a different row. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), tmp=20)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), tmp=20)`}); err != nil { t.Fatal(err) } else if res := res.Results[0].(bool); !res { t.Fatalf("unexpected set row result: %+v", res) } // Ensure the row was populated. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(tmp=20)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(tmp=20)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } // Store row 10 into a table which doesn't exist. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), nonexistent=20)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), nonexistent=20)`}); err != nil { t.Fatal(err) } else if res := res.Results[0].(bool); !res { t.Fatalf("unexpected set row result: %+v", res) } // Ensure the row was populated. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(nonexistent=20)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(nonexistent=20)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) @@ -4200,7 +4200,7 @@ func TestExecutor_Execute_SetRow(t *testing.T) { t.Run("Set_NoSource", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { @@ -4208,7 +4208,7 @@ func TestExecutor_Execute_SetRow(t *testing.T) { } // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10), @@ -4216,35 +4216,35 @@ func TestExecutor_Execute_SetRow(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } // Store row 9 (which doesn't exist) into a different row. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=20)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=20)`}); err != nil { t.Fatal(err) } else if res := res.Results[0].(bool); !res { t.Fatalf("unexpected set row result: %+v", res) } // Ensure the row was populated. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) } // Store row 9 (which doesn't exist) into a row that does exist. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=10)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=9), f=10)`}); err != nil { t.Fatal(err) } else if res := res.Results[0].(bool); !res { t.Fatalf("unexpected set row result: %+v", res) } // Ensure the row was populated. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=10)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{}) { t.Fatalf("unexpected columns: %+v", bits) @@ -4253,7 +4253,7 @@ func TestExecutor_Execute_SetRow(t *testing.T) { t.Run("Set_ExistingDestination", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { @@ -4261,7 +4261,7 @@ func TestExecutor_Execute_SetRow(t *testing.T) { } // Set bits. - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `` + fmt.Sprintf("Set(%d, f=%d)\n", 3, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth-1, 10) + fmt.Sprintf("Set(%d, f=%d)\n", ShardWidth+1, 10) + @@ -4271,21 +4271,21 @@ func TestExecutor_Execute_SetRow(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) } // Store row 10 into an existing row. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), f=20)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Store(Row(f=10), f=20)`}); err != nil { t.Fatal(err) } else if res := res.Results[0].(bool); !res { t.Fatalf("unexpected set row result: %+v", res) } // Ensure the row was populated. - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(f=20)`}); err != nil { t.Fatal(err) } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth - 1, ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", bits) @@ -4296,7 +4296,7 @@ func TestExecutor_Execute_SetRow(t *testing.T) { func benchmarkExistence(nn bool, b *testing.B) { c := test.MustNewCluster(b, 1) var err error - c[0].Config.DataDir, err = ioutil.TempDir(*TempDir, "benchmarkExistence") + c.GetNode(0).Config.DataDir, err = testhook.TempDirInDir(b, *TempDir, "benchmarkExistence") if err != nil { b.Fatalf("getting temp dir: %v", err) } @@ -4305,7 +4305,7 @@ func benchmarkExistence(nn bool, b *testing.B) { b.Fatalf("starting cluster: %v", err) } defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) indexName := "i" fieldName := "f" @@ -4331,7 +4331,7 @@ func benchmarkExistence(nn bool, b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - if err := c[0].API.Import(context.Background(), req); err != nil { + if err := c.GetNode(0).API.Import(context.Background(), req); err != nil { b.Fatal(err) } } @@ -4810,7 +4810,7 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - r, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + r, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: test.query, }) @@ -4832,7 +4832,7 @@ func TestExecutor_GroupByStrings(t *testing.T) { c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "vv", pilosa.OptFieldTypeInt(0, 1000)) c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "nv", pilosa.OptFieldTypeInt(-1000, 1000)) - if err := c[0].API.Import(context.Background(), &pilosa.ImportRequest{ + if err := c.GetNode(0).API.Import(context.Background(), &pilosa.ImportRequest{ Index: "istring", Field: "generals", Shard: 0, @@ -4844,7 +4844,7 @@ func TestExecutor_GroupByStrings(t *testing.T) { var v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 int64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 var nv1, nv2, nv3, nv4 int64 = -1, -2, -3, -4 - if err := c[0].API.ImportValue(context.Background(), &pilosa.ImportValueRequest{ + if err := c.GetNode(0).API.ImportValue(context.Background(), &pilosa.ImportValueRequest{ Index: "istring", Field: "v", Shard: 0, @@ -4854,7 +4854,7 @@ func TestExecutor_GroupByStrings(t *testing.T) { t.Fatalf("importing: %v", err) } - if err := c[0].API.ImportValue(context.Background(), &pilosa.ImportValueRequest{ + if err := c.GetNode(0).API.ImportValue(context.Background(), &pilosa.ImportValueRequest{ Index: "istring", Field: "vv", Shard: 0, @@ -4864,7 +4864,7 @@ func TestExecutor_GroupByStrings(t *testing.T) { t.Fatalf("importing: %v", err) } - if err := c[0].API.ImportValue(context.Background(), &pilosa.ImportValueRequest{ + if err := c.GetNode(0).API.ImportValue(context.Background(), &pilosa.ImportValueRequest{ Index: "istring", Field: "nv", Shard: 0, @@ -5039,7 +5039,7 @@ func TestExecutor_GroupByStrings(t *testing.T) { for i, tst := range tests { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { - r, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + r, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "istring", Query: tst.query, }) @@ -5056,12 +5056,12 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{Keys: true}) + _, err := c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{Keys: true}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldKeys()) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldKeys()) if err != nil { t.Fatalf("creating field: %v", err) } @@ -5079,7 +5079,7 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { } } - _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + _, err = c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: query.String(), }) @@ -5175,7 +5175,7 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) { - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { t.Fatal(err) } else { rows := res.Results[0].(pilosa.RowIdentifiers) @@ -5305,14 +5305,14 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {110, 2}, {110, 0}, }) - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(0, v=10)`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(0, v=10)`}); err != nil { t.Fatal(err) - } else if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, v=100)`}); err != nil { + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, v=100)`}); err != nil { t.Fatal(err) } t.Run("No Field List Arguments", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy()`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy()`}); err != nil { if !strings.Contains(err.Error(), "need at least one child call") { t.Fatalf("unexpected error: \"%v\"", err) } @@ -5320,7 +5320,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("Unknown Field ", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(missing))`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(missing))`}); err != nil { if errors.Cause(err) != pilosa.ErrFieldNotFound { t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldNotFound, err) } @@ -5663,7 +5663,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { func BenchmarkGroupBy(b *testing.B) { c := test.MustNewCluster(b, 1) var err error - c[0].Config.DataDir, err = ioutil.TempDir(*TempDir, "benchmarkGroupBy") + c.GetNode(0).Config.DataDir, err = testhook.TempDirInDir(b, *TempDir, "benchmarkGroupBy-") if err != nil { b.Fatalf("getting temp dir: %v", err) } @@ -5732,7 +5732,7 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", *indexOptions) defer index.Close() _, err := index.CreateField("f", fieldOption...) @@ -5740,7 +5740,7 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt t.Fatal(err) } if writeQuery != "" { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: writeQuery, }); err != nil { @@ -5750,7 +5750,7 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt responses := []pilosa.QueryResponse{} for _, query := range readQueries { - res, err := c[0].API.Query(context.Background(), + res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: query, @@ -5772,16 +5772,16 @@ func TestExecutor_Execute_Shift(t *testing.T) { t.Run("Shift Bit 0", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 0) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { t.Fatalf("unexpected columns: %+v", columns) @@ -5791,10 +5791,10 @@ func TestExecutor_Execute_Shift(t *testing.T) { t.Run("Shift container boundary", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 65535) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{65536}) { t.Fatalf("unexpected columns: %+v", columns) @@ -5804,7 +5804,7 @@ func TestExecutor_Execute_Shift(t *testing.T) { t.Run("Shift shard boundary", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) orig := []uint64{1, ShardWidth - 1, ShardWidth + 1} shift1 := []uint64{2, ShardWidth, ShardWidth + 2} @@ -5814,19 +5814,19 @@ func TestExecutor_Execute_Shift(t *testing.T) { hldr.SetBit("i", "general", 10, bit) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, shift1) { t.Fatalf("unexpected shift by 1: expected: %+v, but got: %+v", shift1, columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=2)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=2)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, shift2) { t.Fatalf("unexpected shift by 2: expected: %+v, but got: %+v", shift2, columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, orig) { t.Fatalf("unexpected shift by 0: expected: %+v, but got: %+v", orig, columns) @@ -5836,19 +5836,19 @@ func TestExecutor_Execute_Shift(t *testing.T) { t.Run("Shift shard boundary no create", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, ShardWidth-2) //shardwidth -1 hldr.SetBit("i", "general", 10, ShardWidth-1) //shardwidth hldr.SetBit("i", "general", 10, ShardWidth) //shardwidth +1 hldr.SetBit("i", "general", 10, ShardWidth+2) //shardwidth +3 exp := []uint64{ShardWidth - 1, ShardWidth, ShardWidth + 1, ShardWidth + 3} - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, exp) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2, ShardWidth + 4}) { t.Fatalf("unexpected columns: \n%+v\n%+v", columns, exp) @@ -5860,7 +5860,7 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { t.Run("results-ids", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 1) hldr.SetBit("i", "general", 10, ShardWidth) hldr.SetBit("i", "general", 10, 2*ShardWidth) @@ -5877,7 +5877,7 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { {(2 * ShardWidth) + 1, false}, } { t.Run(fmt.Sprint(i), func(t *testing.T) { - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf("IncludesColumn(Row(general=10), column=%d)", tt.col)}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf("IncludesColumn(Row(general=10), column=%d)", tt.col)}); err != nil { t.Fatal(err) } else if tt.expIncluded && !res.Results[0].(bool) { t.Fatalf("expected to find column: %d", tt.col) @@ -5890,8 +5890,8 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { t.Run("results-keys", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - cmd := c[0] - hldr := test.Holder{Holder: c[0].Server.Holder()} + cmd := c.GetNode(0) + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) if _, err := index.CreateField("general", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { t.Fatal(err) @@ -5918,7 +5918,7 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { {"twentytwo", false}, } { t.Run(fmt.Sprint(i), func(t *testing.T) { - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf("IncludesColumn(Row(general=ten), column=%s)", tt.col)}); err != nil { + if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf("IncludesColumn(Row(general=ten), column=%s)", tt.col)}); err != nil { t.Fatal(err) } else if tt.expIncluded && !res.Results[0].(bool) { t.Fatalf("expected to find column: %s", tt.col) @@ -5931,12 +5931,12 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { t.Run("errors", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) hldr.SetBit("i", "general", 10, 1) t.Run("no column", func(t *testing.T) { expErr := "IncludesColumn call must specify a column" - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `IncludesColumn(Row(general=10))`}); err == nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `IncludesColumn(Row(general=10))`}); err == nil { t.Fatalf("expected to get an error") } else if !strings.Contains(err.Error(), expErr) { t.Fatalf("expected error: %s, but got: %s", expErr, err.Error()) @@ -5945,7 +5945,7 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { t.Run("no row query", func(t *testing.T) { expErr := "IncludesColumn call must specify a row query" - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `IncludesColumn(column=1)`}); err == nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `IncludesColumn(column=1)`}); err == nil { t.Fatalf("expected to get an error") } else if !strings.Contains(err.Error(), expErr) { t.Fatalf("expected error: %s, but got: %s", expErr, err.Error()) @@ -5957,7 +5957,7 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -5976,7 +5976,7 @@ func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { t.Fatal(err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=3) Set(1, f=3) Set(2, f=4) @@ -6015,7 +6015,7 @@ func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { } else { pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) @@ -6038,7 +6038,7 @@ func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { } else { pql = fmt.Sprintf(`Min(%s, field=dec)`, tt.filter) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{DecimalVal: &tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result.Results[0])) @@ -6061,7 +6061,7 @@ func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { } else { pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) @@ -6085,7 +6085,7 @@ func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { } else { pql = fmt.Sprintf(`Max(%s, field=dec)`, tt.filter) } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + if result, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{DecimalVal: &tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result.Results[0])) @@ -6096,14 +6096,14 @@ func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { t.Run("MinMaxRangeError", func(t *testing.T) { // Min pql := `Set(4, dec=-92233720368547758.08)` - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err == nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err == nil { t.Fatalf("expected error but got: nil") } else if errors.Cause(err) != pilosa.ErrDecimalOutOfRange { t.Fatalf("expected error: %s, but got: %s", pilosa.ErrDecimalOutOfRange, err) } // Max pql = `Set(4, dec=92233720368547758.07)` - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err == nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err == nil { t.Fatalf("expected error but got: nil") } else if errors.Cause(err) != pilosa.ErrDecimalOutOfRange { t.Fatalf("expected error: %s, but got: %s", pilosa.ErrDecimalOutOfRange, err) @@ -6116,14 +6116,14 @@ func TestExecutor_Execute_NoIndex(t *testing.T) { indexOptions := &pilosa.IndexOptions{} c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := c.GetHolder(0) index := hldr.MustCreateIndexIfNotExists("i", *indexOptions) _, err := index.CreateField("f") if err != nil { t.Fatal("should work") } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", Query: "Count(Distinct(Row(gpu_tag='GTX'), index=systems, field=jarvis_id))", }); errors.Cause(err) != pilosa.ErrIndexNotFound { @@ -6140,7 +6140,7 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - api := c[0].API + api := c.GetNode(0).API schema := &pilosa.Schema{} if err := json.NewDecoder(bytes.NewReader(data)).Decode(schema); err != nil { @@ -6274,7 +6274,7 @@ func TestExecutor_Execute_TopNDistinct(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - api := c[0].API + api := c.GetNode(0).API schema := &pilosa.Schema{} if err := json.NewDecoder(bytes.NewReader(data)).Decode(schema); err != nil { @@ -6349,7 +6349,7 @@ func TestTimelessClearRegression(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - api := c[0].API + api := c.GetNode(0).API schema := &pilosa.Schema{} if err := json.NewDecoder(bytes.NewReader(data)).Decode(schema); err != nil { diff --git a/field.go b/field.go index 704f05ae9..e607a8f6d 100644 --- a/field.go +++ b/field.go @@ -35,6 +35,7 @@ import ( "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -607,9 +608,11 @@ func (f *Field) Open() error { return err } + _ = testhook.Opened(f.holder.Auditor, f, nil) f.holder.Logger.Debugf("successfully opened field index/field: %s/%s", f.index, f.name) return nil } + func blockingWriteAvailableShards(fieldPath string, availableShardBytes []byte) { path := filepath.Join(fieldPath, ".available.shards") // Create a temporary file to save to. @@ -972,12 +975,14 @@ func (f *Field) applyOptions(opt FieldOptions) error { func (f *Field) Close() error { f.mu.Lock() defer f.mu.Unlock() + defer func() { + _ = testhook.Closed(f.holder.Auditor, f, nil) + }() // Shutdown the available shards writer if f.doneChan != nil { - f.doneChan <- struct{}{} + close(f.doneChan) f.wg.Wait() close(f.availableShardChan) - close(f.doneChan) f.availableShardChan = nil f.doneChan = nil } diff --git a/field_internal_test.go b/field_internal_test.go index e963841e4..9c8bdd7bf 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -16,7 +16,6 @@ package pilosa import ( "fmt" - "io/ioutil" "math" "os" "path/filepath" @@ -28,6 +27,7 @@ import ( "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/testhook" ) // Ensure a bsiGroup can adjust to its baseValue. @@ -197,11 +197,13 @@ func TestField_DeleteView(t *testing.T) { // TestField represents a test wrapper for Field. type TestField struct { *Field + parent *Index + tb testing.TB } // NewTestField returns a new instance of TestField d/0. func NewTestField(t *testing.T, opts FieldOption) *TestField { - path, err := ioutil.TempDir(*TempDir, "pilosa-field-") + path, err := testhook.TempDirInDir(t, *TempDir, "pilosa-field-") if err != nil { t.Fatal(err) } @@ -211,20 +213,20 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField { if err != nil { panic(err) } - field, err := NewField(h, path, "i", "f", opts) + field, err := idx.CreateField("f", opts) if err != nil { t.Fatal(err) } - field.idx = idx - return &TestField{Field: field} + tf := &TestField{Field: field, parent: idx, tb: t} + testhook.Cleanup(t, func() { + h.Close() + }) + return tf } // OpenField returns a new, opened field at a temporary path. func OpenField(t *testing.T, opts FieldOption) *TestField { f := NewTestField(t, opts) - if err := f.Open(); err != nil { - t.Fatal(err) - } return f } @@ -239,27 +241,16 @@ func (f *TestField) Close() error { // Reopen closes the index and reopens it. func (f *TestField) Reopen() error { - var err error - if err := f.Field.Close(); err != nil { + name := f.Field.Name() + if err := f.parent.Close(); err != nil { + f.parent = nil return err } - - path, index, name := f.Path(), f.Index(), f.Name() - h := NewHolder(DefaultPartitionN) - h.Path = path - idx, err := h.CreateIndex(index, IndexOptions{}) - if err != nil { - return err - } - f.Field, err = NewField(h, path, index, name, OptFieldTypeDefault()) - if err != nil { - return err - } - f.Field.idx = idx - - if err := f.Open(); err != nil { + if err := f.parent.Open(false); err != nil { + f.parent = nil return err } + f.Field = f.parent.Field(name) return nil } diff --git a/field_test.go b/field_test.go index 2fc804b7c..b15e4da74 100644 --- a/field_test.go +++ b/field_test.go @@ -15,7 +15,6 @@ package pilosa_test import ( - "io/ioutil" "math" "testing" @@ -23,6 +22,7 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/test" + "github.com/pilosa/pilosa/v2/testhook" ) var panicOn = pilosa.PanicOn @@ -30,7 +30,7 @@ var panicOn = pilosa.PanicOn // Ensure a field can set & read a bsiGroup value. func TestField_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - idx := test.MustOpenIndex() + idx := test.MustOpenIndex(t) defer idx.Close() f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) @@ -67,7 +67,7 @@ func TestField_SetValue(t *testing.T) { }) t.Run("Overwrite", func(t *testing.T) { - idx := test.MustOpenIndex() + idx := test.MustOpenIndex(t) defer idx.Close() f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) @@ -103,7 +103,7 @@ func TestField_SetValue(t *testing.T) { }) t.Run("ErrBSIGroupNotFound", func(t *testing.T) { - idx := test.MustOpenIndex() + idx := test.MustOpenIndex(t) defer idx.Close() f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()) @@ -121,7 +121,7 @@ func TestField_SetValue(t *testing.T) { }) t.Run("ErrBSIGroupValueTooLow", func(t *testing.T) { - idx := test.MustOpenIndex() + idx := test.MustOpenIndex(t) defer idx.Close() f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) @@ -139,7 +139,7 @@ func TestField_SetValue(t *testing.T) { }) t.Run("ErrBSIGroupValueTooHigh", func(t *testing.T) { - idx := test.MustOpenIndex() + idx := test.MustOpenIndex(t) defer idx.Close() f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(20, 30)) @@ -158,7 +158,7 @@ func TestField_SetValue(t *testing.T) { } func TestField_NameRestriction(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-field-") + path, err := testhook.TempDir(t, "pilosa-field-") if err != nil { panic(err) } @@ -190,7 +190,7 @@ func TestField_NameValidation(t *testing.T) { "charact23112345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901", } - path, err := ioutil.TempDir("", "pilosa-field-") + path, err := testhook.TempDir(t, "pilosa-field-") if err != nil { panic(err) } @@ -210,7 +210,7 @@ func TestField_NameValidation(t *testing.T) { // Ensure can update and delete available shards. func TestField_AvailableShards(t *testing.T) { - idx := test.MustOpenIndex() + idx := test.MustOpenIndex(t) defer idx.Close() f, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()) @@ -253,7 +253,7 @@ func TestField_AvailableShards(t *testing.T) { func TestField_ClearValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - idx := test.MustOpenIndex() + idx := test.MustOpenIndex(t) defer idx.Close() f, err := idx.CreateField("f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) diff --git a/fragment.go b/fragment.go index 09a4bde3e..832d02f32 100644 --- a/fragment.go +++ b/fragment.go @@ -45,6 +45,7 @@ import ( "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" "github.com/pilosa/pilosa/v2/stats" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" ) @@ -170,15 +171,6 @@ type fragment struct { stats stats.StatsClient bitmapInfo *roaring.BitmapInfo - - // txTestingOnly: this looks gross. - // Nonetheless, it allowed us to - // integrate Tx into the - // fragment_internal_test.go suite - // and not break the world all at once. - // - // Only for testing, obviously. - txTestingOnly Tx } // newFragment returns a new instance of Fragment. @@ -267,6 +259,7 @@ func (f *fragment) Open() error { } f.open = true + _ = testhook.Opened(f.holder.Auditor, f, nil) f.holder.Logger.Debugf("successfully opened index/field/view/fragment: %s/%s/%s/%d", f.index, f.field, f.view, f.shard) return nil } @@ -513,6 +506,9 @@ func (f *fragment) openCache() error { func (f *fragment) Close() error { f.mu.Lock() defer f.mu.Unlock() + defer func() { + _ = testhook.Closed(f.holder.Auditor, f, nil) + }() for f.snapshotPending { f.snapshotCond.Wait() } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index d033a24e6..1afa458f5 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -38,6 +38,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" ) @@ -52,13 +53,9 @@ var ( // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set bits on the fragment. if _, err := f.setBit(tx, 120, 1); err != nil { t.Fatal(err) @@ -96,14 +93,10 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set and then clear bits on the fragment. if _, err := f.setBit(tx, 1000, 1); err != nil { t.Fatal(err) @@ -142,11 +135,7 @@ func TestFragment_ClearBit(t *testing.T) { // What about rowcache timing. func TestFragment_RowcacheMap(t *testing.T) { var done int64 - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") - _ = idx - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() + f, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") // Under -race, this test turns out to take a fairly long time // to run with larger OpN, because we write 50,000 bits to @@ -192,15 +181,10 @@ func TestFragment_RowcacheMap(t *testing.T) { // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { notBlueGreenTest(t) - - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set and then clear bits on the fragment. if _, err := f.setBit(tx, 1000, 1); err != nil { t.Fatal(err) @@ -229,13 +213,11 @@ func TestFragment_ClearRow(t *testing.T) { // Ensure a fragment can set a row. func TestFragment_SetRow(t *testing.T) { notBlueGreenTest(t) - f, idx := mustOpenFragment("i", "f", viewStandard, 7, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 7, "") _ = idx defer f.Clean(t) // Obtain transction. - tx := f.txTestingOnly - defer tx.Rollback() rowID := uint64(1000) @@ -291,14 +273,10 @@ func TestFragment_SetRow(t *testing.T) { // Ensure a fragment can set & read a value. func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set value. if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { t.Fatal(err) @@ -347,14 +325,10 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("Overwrite", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set value. if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { t.Fatal(err) @@ -396,14 +370,10 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("Clear", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set value. if changed, err := f.setValue(tx, 100, 16, 3829); err != nil { t.Fatal(err) @@ -444,14 +414,10 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("NotExists", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set value. if changed, err := f.setValue(tx, 100, 10, 20); err != nil { t.Fatal(err) @@ -480,14 +446,10 @@ func TestFragment_SetValue(t *testing.T) { values[i] = values[i] % (1 << bitDepth) } - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set values. m := make(map[uint64]int64) for _, value := range values { @@ -542,13 +504,9 @@ func TestFragment_SetValue(t *testing.T) { func TestFragment_Sum(t *testing.T) { const bitDepth = 16 - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set values. vals := []struct { cid uint64 @@ -618,13 +576,9 @@ func TestFragment_Sum(t *testing.T) { func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) @@ -705,14 +659,10 @@ func TestFragment_Range(t *testing.T) { const bitDepth = 16 t.Run("EQ", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) @@ -733,14 +683,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("EQOversizeRegression", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set values. if _, err := f.setValue(tx, 1000, 1, 0); err != nil { t.Fatal(err) @@ -762,14 +708,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("NEQ", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) @@ -790,14 +732,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("LT", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) @@ -843,14 +781,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("LTRegression", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - if _, err := f.setValue(tx, 1, 1, 1); err != nil { t.Fatal(err) } @@ -863,14 +797,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("LTMaxRegression", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - if _, err := f.setValue(tx, 1, 2, 3); err != nil { t.Fatal(err) } else if _, err := f.setValue(tx, 2, 2, 0); err != nil { @@ -885,14 +815,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("GT", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) @@ -938,14 +864,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("GTMinRegression", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - if _, err := f.setValue(tx, 1, 2, 0); err != nil { t.Fatal(err) } else if _, err := f.setValue(tx, 2, 2, 1); err != nil { @@ -960,14 +882,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("GTOversizeRegression", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - if _, err := f.setValue(tx, 1, 2, 0); err != nil { t.Fatal(err) } else if _, err := f.setValue(tx, 2, 2, 1); err != nil { @@ -982,14 +900,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { t.Fatal(err) @@ -1035,14 +949,10 @@ func TestFragment_Range(t *testing.T) { }) t.Run("BetweenCommonBitsRegression", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - if _, err := f.setValue(tx, 1, 64, 0xf0); err != nil { t.Fatal(err) } else if _, err := f.setValue(tx, 2, 64, 0xf1); err != nil { @@ -1059,11 +969,7 @@ func TestFragment_Range(t *testing.T) { // benchmarkSetValues is a helper function to explore, very roughly, the cost // of setting values. -func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - +func benchmarkSetValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { column := uint64(0) for i := 0; i < b.N; i++ { // We're not checking the error because this is a benchmark. @@ -1078,17 +984,17 @@ func BenchmarkFragment_SetValue(b *testing.B) { depths := []uint{4, 8, 16} for _, bitDepth := range depths { name := fmt.Sprintf("Depth%d", bitDepth) - f, idx := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + f, idx, tx := mustOpenFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0, "none") _ = idx b.Run(name+"_Sparse", func(b *testing.B) { - benchmarkSetValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) + benchmarkSetValues(b, tx, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) }) f.Clean(b) - f, idx = mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + f, idx, tx = mustOpenFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0, "none") _ = idx b.Run(name+"_Dense", func(b *testing.B) { - benchmarkSetValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) + benchmarkSetValues(b, tx, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) }) f.Clean(b) } @@ -1096,11 +1002,7 @@ func BenchmarkFragment_SetValue(b *testing.B) { // benchmarkImportValues is a helper function to explore, very roughly, the cost // of setting values using the special setter used for imports. -func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - +func benchmarkImportValues(b *testing.B, tx Tx, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { column := uint64(0) b.StopTimer() columns := make([]uint64, b.N) @@ -1122,16 +1024,16 @@ func BenchmarkFragment_ImportValue(b *testing.B) { depths := []uint{4, 8, 16} for _, bitDepth := range depths { name := fmt.Sprintf("Depth%d", bitDepth) - f, idx := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) + f, idx, tx := mustOpenBSIFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0) _ = idx b.Run(name+"_Sparse", func(b *testing.B) { - benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) + benchmarkImportValues(b, tx, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) }) f.Clean(b) - f, idx = mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) + f, idx, tx = mustOpenBSIFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0) _ = idx b.Run(name+"_Dense", func(b *testing.B) { - benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) + benchmarkImportValues(b, tx, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) }) f.Clean(b) } @@ -1163,15 +1065,11 @@ func BenchmarkFragment_RepeatedSmallImports(b *testing.B) { updateRows[i] = uint64(rand.Int63n(int64(numRows))) // row id updateCols[i] = uint64(rand.Int63n(ShardWidth)) // column id } - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") _ = idx f.MaxOpN = opN defer f.Clean(b) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - err := f.importRoaringT(tx, getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) if err != nil { b.Fatalf("importing base data for benchmark: %v", err) @@ -1206,12 +1104,10 @@ func BenchmarkFragment_RepeatedSmallImportsRoaring(b *testing.B) { b.StopTimer() // build the update data set all at once - this will get applied // to a fragment in numUpdates batches - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") _ = idx f.MaxOpN = opN defer f.Clean(b) - tx := f.txTestingOnly - defer tx.Rollback() err := f.importRoaringT(tx, getZipfRowsSliceRoaring(numRows, 1, 0, ShardWidth), false) if err != nil { @@ -1259,14 +1155,9 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { b.Run(fmt.Sprintf("Updates%dVals%dOpN%d", numUpdates, valsPerUpdate, opN), func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - f, idx := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) - _ = idx + f, _, tx := mustOpenBSIFragment(b, "i", "f", viewBSIGroupPrefix+"foo", 0) f.MaxOpN = opN - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - err := f.importValue(tx, initialCols, initialVals, 21, false) if err != nil { b.Fatalf("initial value import: %v", err) @@ -1294,13 +1185,9 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set and then clear bits on the fragment. if _, err := f.setBit(tx, 1000, 1); err != nil { t.Fatal(err) @@ -1330,14 +1217,10 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set bits on the fragment. if _, err := f.setBit(tx, 100, 20); err != nil { t.Fatal(err) @@ -1364,14 +1247,10 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set bits on the rows 100, 101, & 102. f.mustSetBits(tx, 100, 1, 3, 200) f.mustSetBits(tx, 101, 1) @@ -1392,13 +1271,9 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set bits on the rows 100, 101, & 102. f.mustSetBits(tx, 100, 1, 3, 200) f.mustSetBits(tx, 101, 1) @@ -1436,14 +1311,10 @@ func TestFragment_Top_Filter(t *testing.T) { // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Create an intersecting input row. src := NewRow(1, 2, 3) @@ -1472,14 +1343,10 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { t.Skip("short mode") } - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Create an intersecting input row. src := NewRow( 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, @@ -1525,14 +1392,10 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set bits on various rows. f.mustSetBits(tx, 100, 1, 2, 3) f.mustSetBits(tx, 101, 4, 5, 6, 7) @@ -1551,14 +1414,10 @@ func TestFragment_TopN_IDs(t *testing.T) { // Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeNone) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeNone) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set bits on various rows. f.mustSetBits(tx, 100, 1, 2, 3) f.mustSetBits(tx, 101, 4, 5, 6, 7) @@ -1578,7 +1437,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { cacheSize := uint32(3) // Create Index. - index := mustOpenIndex(IndexOptions{}) + index := mustOpenIndex(t, IndexOptions{}) defer index.Close() // Create field. @@ -1641,14 +1500,10 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Retrieve checksum and set bits. orig, err := f.Checksum() if err != nil { @@ -1671,13 +1526,10 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - // Retrieve initial checksum. var prev []FragmentBlock @@ -1710,7 +1562,7 @@ func TestFragment_Blocks(t *testing.T) { // Set bit on different column. tx = idx.Txf.NewTx(Txo{Write: true, Index: idx, Fragment: f}) - f.txTestingOnly = tx // let the Clean do the Rollback + defer tx.Rollback() if _, err := f.setBit(tx, 20, 100); err != nil { t.Fatal(err) } @@ -1725,14 +1577,10 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set bits on a different block. if _, err := f.setBit(tx, 100, 1); err != nil { t.Fatal(err) @@ -1751,14 +1599,10 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeLRU) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeLRU) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { if _, err := f.setBit(tx, i, 0); err != nil { @@ -1792,7 +1636,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { func TestFragment_RankCache_Persistence(t *testing.T) { roaringOnlyTest(t) - index := mustOpenIndex(IndexOptions{}) + index := mustOpenIndex(t, IndexOptions{}) defer index.Close() // Create field. @@ -1867,13 +1711,9 @@ func roaringOnlyBenchmark(b *testing.B) { func TestFragment_WriteTo_ReadFrom(t *testing.T) { roaringOnlyTest(t) - f0, idx := mustOpenFragment("i", "f", viewStandard, 0, "") - _ = idx + f0, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f0.Clean(t) - tx := f0.txTestingOnly - defer tx.Rollback() - // Set and then clear bits on the fragment. if _, err := f0.setBit(tx, 1000, 1); err != nil { t.Fatal(err) @@ -1896,8 +1736,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1, idx := mustOpenFragment("i", "f", viewStandard, 0, "") - _ = idx + f1, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f1.Clean(t) if rn, err := f1.ReadFrom(&buf); err != nil { // eventually calls fragment.fillFragmentFromArchive @@ -1936,7 +1775,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.CleanKeep(b) + defer f.Clean(b) // Reset timer and execute benchmark. b.ResetTimer() @@ -1950,14 +1789,10 @@ func BenchmarkFragment_Blocks(b *testing.B) { } func BenchmarkFragment_IntersectionCount(b *testing.B) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") defer f.Clean(b) f.MaxOpN = math.MaxInt32 - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Generate some intersecting data. for i := 0; i < 10000; i += 2 { if _, err := f.setBit(tx, 1, uint64(i)); err != nil { @@ -1989,14 +1824,10 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { } func TestFragment_Tanimoto(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. @@ -2017,14 +1848,10 @@ func TestFragment_Tanimoto(t *testing.T) { } func TestFragment_Zero_Tanimoto(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - src := NewRow(1, 2, 3) // Set bits on the rows 100, 101, & 102. @@ -2049,14 +1876,10 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { func TestFragment_Snapshot_Run(t *testing.T) { roaringOnlyTest(t) - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set bits on the fragment. for i := uint64(1); i < 3; i++ { if _, err := f.setBit(tx, 1000, i); err != nil { @@ -2081,14 +1904,9 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Ensure a fragment can set mutually exclusive values. func TestFragment_SetMutex(t *testing.T) { - f, idx := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - _ = idx + f, _, tx := mustOpenMutexFragment(t, "i", "f", viewStandard, 0, "") defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - var cols []uint64 // Set a value on column 100. @@ -2200,14 +2018,10 @@ func TestFragment_ImportSet(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set import. err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { @@ -2322,14 +2136,10 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set import. err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { @@ -2375,14 +2185,10 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) { func TestFragment_ConcurrentImport(t *testing.T) { t.Run("bulkImportStandard", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - eg := errgroup.Group{} eg.Go(func() error { return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) eg.Go(func() error { return f.bulkImportStandard(tx, []uint64{3, 4}, []uint64{3, 4}, &ImportOptions{}) }) @@ -2477,14 +2283,9 @@ func TestFragment_ImportMutex(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { - f, idx := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - _ = idx + f, _, tx := mustOpenMutexFragment(t, "i", "f", viewStandard, 0, "") defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set import. err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { @@ -2601,13 +2402,9 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { - f, idx := mustOpenMutexFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenMutexFragment(t, "i", "f", viewStandard, 0, "") defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set import. err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { @@ -2736,14 +2533,9 @@ func TestFragment_ImportBool(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { - f, idx := mustOpenBoolFragment("i", "f", viewStandard, 0, "") - _ = idx + f, _, tx := mustOpenBoolFragment(t, "i", "f", viewStandard, 0, "") defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set import. err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { @@ -2860,14 +2652,9 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { - f, idx := mustOpenBoolFragment("i", "f", viewStandard, 0, "") - _ = idx + f, idx, tx := mustOpenBoolFragment(t, "i", "f", viewStandard, 0, "") defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Set import. err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) if err != nil { @@ -2922,7 +2709,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.CleanKeep(b) + defer f.Clean(b) b.ResetTimer() // Reset timer and execute benchmark. @@ -2937,8 +2724,9 @@ func BenchmarkFragment_Snapshot(b *testing.B) { } func BenchmarkFragment_FullSnapshot(b *testing.B) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") _ = idx + tx.Rollback() defer f.Clean(b) // Generate some intersecting data. @@ -3007,11 +2795,8 @@ func BenchmarkFragment_Import(b *testing.B) { // since bulkImport modifies the input slices, we make new copies for each round copy(rowsUse, rows) copy(colsUse, cols) - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, "") _ = idx - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() b.StartTimer() if err := f.bulkImport(tx, rowsUse, colsUse, options); err != nil { b.Errorf("Error Building Sample: %s", err) @@ -3037,10 +2822,8 @@ func BenchmarkImportRoaring(b *testing.B) { b.Run(fmt.Sprintf("Rows%dCache_%s", numRows, cacheType), func(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { - f, _ := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) + f, _, tx := mustOpenFragment(b, "i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) b.StartTimer() - tx := f.txTestingOnly - defer tx.Rollback() err := f.importRoaringT(tx, data, false) if err != nil { @@ -3077,19 +2860,19 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { b.Run(fmt.Sprintf("Rows%dConcurrency%dCache_%s", numRows, concurrency, cacheType), func(b *testing.B) { b.StopTimer() frags := make([]*fragment, concurrency) + txs := make([]Tx, concurrency) for i := 0; i < b.N; i++ { for j := 0; j < concurrency; j++ { - frags[j], _ = mustOpenFragment("i", "f", viewStandard, uint64(j), cacheType) + frags[j], _, txs[j] = mustOpenFragment(b, "i", "f", viewStandard, uint64(j), cacheType) } eg := errgroup.Group{} b.StartTimer() for j := 0; j < concurrency; j++ { j := j eg.Go(func() error { - tx := frags[j].txTestingOnly - defer tx.Rollback() + defer txs[j].Rollback() - err := frags[j].importRoaringT(tx, data[j], false) + err := frags[j].importRoaringT(txs[j], data[j], false) // error unimportant if it happened, but we want // any snapshots to have finished. _ = defaultSnapshotQueue.Await(frags[j]) @@ -3124,9 +2907,10 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { b.Run(fmt.Sprintf("Rows%dCols%dConcurrency%dCache_%s", numRows, numCols, concurrency, cacheType), func(b *testing.B) { b.StopTimer() frags := make([]*fragment, concurrency) + txs := make([]Tx, concurrency) for i := 0; i < b.N; i++ { for j := 0; j < concurrency; j++ { - frags[j], _ = mustOpenFragment("i", "f", viewStandard, uint64(j), cacheType) + frags[j], _, txs[j] = mustOpenFragment(b, "i", "f", viewStandard, uint64(j), cacheType) // the cost of actually doing the op log for the large initial data set // is excessive. force storage into snapshotted state, then use import @@ -3146,10 +2930,9 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { for j := 0; j < concurrency; j++ { j := j eg.Go(func() error { - tx := frags[j].txTestingOnly - defer tx.Rollback() + defer txs[j].Rollback() - err := frags[j].importRoaringT(tx, updata, false) + err := frags[j].importRoaringT(txs[j], updata, false) err2 := defaultSnapshotQueue.Await(frags[j]) if err == nil { err = err2 @@ -3183,12 +2966,8 @@ func BenchmarkImportStandard(b *testing.B) { for i := 0; i < b.N; i++ { copy(rowIDs, rowIDsOrig) copy(columnIDs, columnIDsOrig) - f, idx := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) + f, idx, tx := mustOpenFragment(b, "i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) _ = idx - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - b.StartTimer() err := f.bulkImport(tx, rowIDs, columnIDs, &ImportOptions{}) if err != nil { @@ -3216,10 +2995,8 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { b.Run(name, func(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { - f, idx := mustOpenFragment("i", fmt.Sprintf("r%dc%dcache_%s", numRows, numCols, cacheType), viewStandard, 0, cacheType) + f, idx, tx := mustOpenFragment(b, "i", fmt.Sprintf("r%dc%dcache_%s", numRows, numCols, cacheType), viewStandard, 0, cacheType) _ = idx - tx := f.txTestingOnly - defer tx.Rollback() // the cost of actually doing the op log for the large initial data set // is excessive. force storage into snapshotted state, then use import @@ -3282,10 +3059,9 @@ func BenchmarkUpdatePathological(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { b.StopTimer() - f, idx := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) + f, idx, tx := mustOpenFragment(b, "i", "f", viewStandard, 0, DefaultCacheType) _ = idx - tx := f.txTestingOnly - defer tx.Rollback() + err := f.importRoaringT(tx, exists, false) if err != nil { b.Fatalf("importing roaring: %v", err) @@ -3302,12 +3078,9 @@ func BenchmarkUpdatePathological(b *testing.B) { var bigFrag string -func initBigFrag() { +func initBigFrag(tb testing.TB) { if bigFrag == "" { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) - _ = idx - tx := f.txTestingOnly - defer tx.Rollback() + f, _, tx := mustOpenFragment(tb, "i", "f", viewStandard, 0, DefaultCacheType) for i := int64(0); i < 10; i++ { // 10 million rows, 1 bit per column, random seeded by i data := getZipfRowsSliceRoaring(10000000, i, 0, ShardWidth) @@ -3327,7 +3100,7 @@ func initBigFrag() { func BenchmarkImportIntoLargeFragment(b *testing.B) { b.StopTimer() - initBigFrag() + initBigFrag(b) rowsOrig, colsOrig := getUpdataSlices(10000000, 11000, 0) rows, cols := make([]uint64, len(rowsOrig)), make([]uint64, len(colsOrig)) opts := &ImportOptions{} @@ -3377,7 +3150,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { b.StopTimer() - initBigFrag() + initBigFrag(b) updata := getUpdataRoaring(10000000, 11000, 0) for i := 0; i < b.N; i++ { origF, err := os.Open(bigFrag) @@ -3396,14 +3169,16 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { fi.Close() // want to do this, but no path argument. - //nf, idx := mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) + //nf, idx, tx := mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) th := newTestHolder() idx := fragTestMustOpenIndex(filepath.Dir(fi.Name()), "i", th, IndexOptions{}) if th.NeedsSnapshot() { th.SnapshotQueue = newSnapshotQueue(1, 1, nil) } + // XXX TODO: newFragment is using the wrong path here, we should fix that someday. nf := newFragment(th, fi.Name(), "i", "f", viewStandard, 0, 0) + defer nf.Clean(b) tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: nf}) defer tx.Rollback() @@ -3418,19 +3193,13 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { if err != nil { b.Fatalf("bulkImport: %v", err) } - - nf.Clean(b) } } func TestGetZipfRowsSliceRoaring(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, DefaultCacheType) _ = idx - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - data := getZipfRowsSliceRoaring(10, 1, 0, ShardWidth) err := f.importRoaringT(tx, data, false) if err != nil { @@ -3595,6 +3364,8 @@ func (f *fragment) sanityCheck(t testing.TB) { } } +// Clean used to delete fragments, but doesn't anymore -- deleting is +// handled by the testhook.TempDir when appropriate. func (f *fragment) Clean(t testing.TB) { f.mu.Lock() // we need to ensure that we unlock the mutex before terminating @@ -3620,24 +3391,11 @@ func (f *fragment) Clean(t testing.TB) { } } }() - if f.txTestingOnly != nil { - f.txTestingOnly.Rollback() - panicOn(f.idx.Txf.CloseIndex(f.idx)) - } errc := f.Close() // prevent double-closes of generation during testing. f.gen = nil - var errf error - if FileExists(f.path) { - errf = os.Remove(f.path) // remove /var/folders/2x/hm9gp5ys3k9gmm5f_vzm_6wc0000gn/T/pilosa-index-768377904/i/f/views/standard/fragments/0: no such file or directory - } - errp := os.Remove(f.cachePath()) - if errc != nil || errf != nil { - t.Fatal("cleaning up fragment: ", errc, errf, errp) - } - // not all fragments have cache files - if errp != nil && !os.IsNotExist(errp) { - t.Fatalf("cleaning up fragment cache: %v", errp) + if errc != nil { + t.Fatalf("error closing fragment: %v", errc) } } @@ -3647,27 +3405,13 @@ func (f *fragment) importRoaringT(tx Tx, data []byte, clear bool) error { return f.importRoaring(context.Background(), tx, data, clear) } -// CleanKeep is just like Clean(), but it doesn't remove the -// fragment file (note that it DOES remove the cache file). -func (f *fragment) CleanKeep(t testing.TB) { - errc := f.Close() - errp := os.Remove(f.cachePath()) - if errc != nil { - t.Fatal("closing fragment: ", errc, errp) - } - // not all fragments have cache files - if errp != nil && !os.IsNotExist(errp) { - t.Fatalf("cleaning up fragment cache: %v", errp) - } -} - // mustOpenFragment returns a new instance of Fragment with a temporary path. -func mustOpenFragment(index, field, view string, shard uint64, cacheType string) (*fragment, *Index) { - return mustOpenFragmentFlags(index, field, view, shard, cacheType, 0) +func mustOpenFragment(tb testing.TB, index, field, view string, shard uint64, cacheType string) (*fragment, *Index, Tx) { + return mustOpenFragmentFlags(tb, index, field, view, shard, cacheType, 0) } -func mustOpenBSIFragment(index, field, view string, shard uint64) (*fragment, *Index) { - return mustOpenFragmentFlags(index, field, view, shard, "", 1) +func mustOpenBSIFragment(tb testing.TB, index, field, view string, shard uint64) (*fragment, *Index, Tx) { + return mustOpenFragmentFlags(tb, index, field, view, shard, "", 1) } func newTestHolder() *Holder { @@ -3694,9 +3438,9 @@ func fragTestMustOpenIndex(holderDir, index string, holder *Holder, opt IndexOpt } // mustOpenFragment returns a new instance of Fragment with a temporary path. -func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) (*fragment, *Index) { +func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint64, cacheType string, flags byte) (*fragment, *Index, Tx) { - holderDir, err := ioutil.TempDir(*TempDir, "holder-dir") + holderDir, err := testhook.TempDirInDir(tb, *TempDir, "holder-dir") panicOn(err) if cacheType == "" { @@ -3704,6 +3448,9 @@ func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType st } th := newTestHolder() + testhook.Cleanup(tb, func() { + th.Close() + }) idx := fragTestMustOpenIndex(holderDir, index, th, IndexOptions{}) if th.NeedsSnapshot() { th.SnapshotQueue = newSnapshotQueue(1, 1, nil) @@ -3715,7 +3462,10 @@ func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType st f := newFragment(th, fragPath, index, field, view, shard, flags) tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) - f.txTestingOnly = tx + testhook.Cleanup(tb, func() { + tx.Rollback() + panicOn(idx.Txf.CloseIndex(idx)) + }) f.CacheType = cacheType f.RowAttrStore = &memAttrStore{ @@ -3725,21 +3475,21 @@ func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType st if err := f.Open(); err != nil { panic(err) } - return f, idx + return f, idx, tx } // mustOpenMutexFragment returns a new instance of Fragment for a mutex field. -func mustOpenMutexFragment(index, field, view string, shard uint64, cacheType string) (*fragment, *Index) { - frag, idx := mustOpenFragment(index, field, view, shard, cacheType) +func mustOpenMutexFragment(tb testing.TB, index, field, view string, shard uint64, cacheType string) (*fragment, *Index, Tx) { + frag, idx, tx := mustOpenFragment(tb, index, field, view, shard, cacheType) frag.mutexVector = newRowsVector(frag) - return frag, idx + return frag, idx, tx } // mustOpenBoolFragment returns a new instance of Fragment for a bool field. -func mustOpenBoolFragment(index, field, view string, shard uint64, cacheType string) (*fragment, *Index) { - frag, idx := mustOpenFragment(index, field, view, shard, cacheType) +func mustOpenBoolFragment(tb testing.TB, index, field, view string, shard uint64, cacheType string) (*fragment, *Index, Tx) { + frag, idx, tx := mustOpenFragment(tb, index, field, view, shard, cacheType) frag.mutexVector = newBoolVector(frag) - return frag, idx + return frag, idx, tx } // Reopen closes the fragment and reopens it as a new instance. @@ -3774,10 +3524,8 @@ func addToBitmap(bm *roaring.Bitmap, rowID uint64, columnIDs ...uint64) { // Test Various methods of retrieving RowIDs func TestFragment_RowsIteration(t *testing.T) { t.Run("firstContainer", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx - tx := f.txTestingOnly - defer tx.Rollback() defer f.Clean(t) expectedAll := make([]uint64, 0) @@ -3808,10 +3556,9 @@ func TestFragment_RowsIteration(t *testing.T) { }) t.Run("secondRow", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx - tx := f.txTestingOnly - defer tx.Rollback() + defer f.Clean(t) expected := []uint64{1, 2} @@ -3843,10 +3590,9 @@ func TestFragment_RowsIteration(t *testing.T) { }) t.Run("combinations", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx - tx := f.txTestingOnly - defer tx.Rollback() + defer f.Clean(t) expectedRows := make([]uint64, 0) @@ -3897,11 +3643,9 @@ func TestFragment_RoaringImport(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) - tx := f.txTestingOnly - defer tx.Rollback() for num, input := range test { buf := &bytes.Buffer{} @@ -3948,13 +3692,10 @@ func TestFragment_RoaringImportTopN(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) _ = idx defer f.Clean(t) - tx := f.txTestingOnly - defer tx.Rollback() - options := &ImportOptions{} err := f.bulkImport(tx, test.rowIDs, test.colIDs, options) if err != nil { @@ -4090,10 +3831,9 @@ func calcExpected(inputs ...[]uint64) [][]uint64 { func TestFragmentRowIterator(t *testing.T) { t.Run("basic", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeRanked) _ = idx - tx := f.txTestingOnly - defer tx.Rollback() + defer f.Clean(t) f.mustSetBits(tx, 0, 0) @@ -4136,11 +3876,9 @@ func TestFragmentRowIterator(t *testing.T) { }) t.Run("skipped rows", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeRanked) _ = idx defer f.Clean(t) - tx := f.txTestingOnly - defer tx.Rollback() f.mustSetBits(tx, 1, 0) f.mustSetBits(tx, 3, 0) @@ -4182,11 +3920,9 @@ func TestFragmentRowIterator(t *testing.T) { }) t.Run("basic wrapped", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeRanked) _ = idx defer f.Clean(t) - tx := f.txTestingOnly - defer tx.Rollback() f.mustSetBits(tx, 0, 0) f.mustSetBits(tx, 1, 0) @@ -4217,11 +3953,8 @@ func TestFragmentRowIterator(t *testing.T) { }) t.Run("skipped rows wrapped", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) - _ = idx + f, _, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeRanked) defer f.Clean(t) - tx := f.txTestingOnly - defer tx.Rollback() f.mustSetBits(tx, 1, 0) f.mustSetBits(tx, 3, 0) @@ -4255,10 +3988,9 @@ func TestFragmentRowIterator(t *testing.T) { // same, with commits func TestFragmentRowIterator_WithTxCommit(t *testing.T) { t.Run("basic", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeRanked) _ = idx - tx := f.txTestingOnly - defer tx.Rollback() + defer f.Clean(t) f.mustSetBits(tx, 0, 0) @@ -4305,11 +4037,9 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { }) t.Run("skipped rows", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeRanked) _ = idx defer f.Clean(t) - tx := f.txTestingOnly - defer tx.Rollback() f.mustSetBits(tx, 1, 0) f.mustSetBits(tx, 3, 0) @@ -4355,11 +4085,9 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { }) t.Run("basic wrapped", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeRanked) _ = idx defer f.Clean(t) - tx := f.txTestingOnly - defer tx.Rollback() f.mustSetBits(tx, 0, 0) f.mustSetBits(tx, 1, 0) @@ -4394,11 +4122,9 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { }) t.Run("skipped rows wrapped", func(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeRanked) _ = idx defer f.Clean(t) - tx := f.txTestingOnly - defer tx.Rollback() f.mustSetBits(tx, 1, 0) f.mustSetBits(tx, 3, 0) @@ -4438,11 +4164,10 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { func TestUnionInPlaceMapped(t *testing.T) { roaringOnlyTest(t) - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) + f, _, _ := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeNone) // note: clean has to be deferred first, because it has to run with // the lock *not* held, because it is sometimes so it has to grab the // lock... - _ = idx defer f.Clean(t) f.mu.Lock() @@ -4514,8 +4239,7 @@ func randPositions(n int, r *rand.Rand) []uint64 { } func TestFragmentPositionsForValue(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) - _ = idx + f, _, _ := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeNone) defer f.Clean(t) tests := []struct { @@ -4597,14 +4321,10 @@ func TestFragmentPositionsForValue(t *testing.T) { } func TestIntLTRegression(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeNone) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - _, err := f.setValue(tx, 1, 6, 33) if err != nil { t.Fatalf("setting value: %v", err) @@ -4631,14 +4351,10 @@ func sliceEq(x, y []uint64) bool { } func TestFragmentBSIUnsigned(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeNone) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Number of bits to test. const k = 6 @@ -4794,14 +4510,10 @@ func TestFragmentBSIUnsigned(t *testing.T) { // same, WithTxCommit version func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeNone) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Number of bits to test. const k = 6 @@ -4960,14 +4672,10 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { } func TestFragmentBSISigned(t *testing.T) { - f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) + f, idx, tx := mustOpenFragment(t, "i", "f", "v", 0, CacheTypeNone) _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // Number of bits to test. const k = 6 @@ -5171,14 +4879,10 @@ func TestImportClearRestart(t *testing.T) { } } - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx f.MaxOpN = maxOpN - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - err := f.bulkImport(tx, testrows, testcols, &ImportOptions{}) if err != nil { t.Fatalf("initial small import: %v", err) @@ -5212,7 +4916,6 @@ func TestImportClearRestart(t *testing.T) { check(t, tx, f, exp) h := NewHolder(DefaultPartitionN) - h.Path = filepath.Dir(f.path) idx2, err := h.CreateIndex("i", IndexOptions{}) _ = idx2 panicOn(err) @@ -5220,7 +4923,7 @@ func TestImportClearRestart(t *testing.T) { // OVERWRITING the f.path with a new fragment f2 := newFragment(h, f.path, "i", "f", viewStandard, 0, 0) - // f2, idx2 := mustOpenFragment("i", "f", viewStandard, 0, "") + // f2, idx2 := mustOpenFragment(t, "i", "f", viewStandard, 0, "") // _ = idx2 f2.MaxOpN = maxOpN @@ -5326,20 +5029,25 @@ func check(t *testing.T, tx Tx, f *fragment, exp map[uint64]map[uint64]struct{}) } func TestImportValueConcurrent(t *testing.T) { - f, idx := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) - - // produces false positives under blue_green because of races - // between commits, and is probematic under single writer - // backends like rbf and lmdb. Marking as roaring-only. - roaringOnlyTest(t) - - // Since eg.Go gets called multiple times below, each - // time needs its own Tx. So close the default one and - // make a new one each time. - tx := f.txTestingOnly + f, idx, tx := mustOpenBSIFragment(t, "i", "f", viewBSIGroupPrefix+"foo", 0) + defer f.Clean(t) + // we will be making a new Tx each time, so we can rollback the default provided one. tx.Rollback() - defer f.Clean(t) + types := idx.Txf.TxTypes() + for _, ty := range types { + switch ty { + case roaringTxn: + t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + + "blueGreenTx because the lack of transactional consistency " + + "from Roaring-per-file will create false comparison " + + "failures.")) + case lmdbTxn: + t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + + "lmdb since only a single writer is allowed at once.")) + } + } + eg := &errgroup.Group{} for i := 0; i < 4; i++ { i := i @@ -5381,15 +5089,10 @@ func TestImportMultipleValues(t *testing.T) { for i, test := range tests { for _, maxOpN := range []int{0, 10000} { // test small/large write t.Run(fmt.Sprintf("%dLowOpN", i), func(t *testing.T) { - f, idx := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) - _ = idx + f, _, tx := mustOpenBSIFragment(t, "i", "f", viewBSIGroupPrefix+"foo", 0) f.MaxOpN = maxOpN defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - err := f.importValue(tx, test.cols, test.vals, test.depth, false) if err != nil { t.Fatalf("importing values: %v", err) @@ -5449,15 +5152,10 @@ func TestImportValueRowCache(t *testing.T) { for i, test := range tests { for _, maxOpN := range []int{1, 10000} { t.Run(fmt.Sprintf("%dMaxOpN%d", i, maxOpN), func(t *testing.T) { - f, idx := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) + f, _, tx := mustOpenBSIFragment(t, "i", "f", viewBSIGroupPrefix+"foo", 0) f.MaxOpN = maxOpN - _ = idx defer f.Clean(t) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // First import (tc1) if err := f.importValue(tx, test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil { t.Fatalf("importing values: %v", err) @@ -5485,14 +5183,12 @@ func TestImportValueRowCache(t *testing.T) { } func TestFragmentConcurrentReadWrite(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) _ = idx defer f.Clean(t) // Obtain transaction, but don't start another b/c the // two goroutines below need the same view. - tx := f.txTestingOnly - defer tx.Rollback() eg := &errgroup.Group{} eg.Go(func() error { @@ -5518,8 +5214,8 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { } func TestRemapCache(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") - _ = idx + f, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") + defer f.Close() index, field, view, shard := f.index, f.field, f.view, f.shard // request a panic that doesn't kill the program on fault @@ -5538,10 +5234,6 @@ func TestRemapCache(t *testing.T) { } }() - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // create a container _, err := tx.Add(index, field, view, shard, !doBatched, 65537) if err != nil { @@ -5580,12 +5272,8 @@ func TestRemapCache(t *testing.T) { } func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { - f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() - // byShardWidth is a map of the same roaring (fragment) data generated // with different shard widths. // TODO: a better approach may be to generate this in the test based diff --git a/gendebug_test.go b/gendebug_test.go index 3dd86fd8f..c42e76801 100644 --- a/gendebug_test.go +++ b/gendebug_test.go @@ -17,23 +17,31 @@ package pilosa import ( + "errors" "fmt" - "os" - "testing" + "runtime" + + "github.com/pilosa/pilosa/v2/testhook" ) -func examineResults() { - results := reportGenerations() +func examineResults() error { + runtime.GC() + stats, results := reportGenerations() + if len(stats) > 0 { + fmt.Printf("generation stats: %s\n", stats) + } + if len(results) == 0 { + return nil + } if len(results) > 0 { fmt.Printf("generations:\n") for _, res := range results { fmt.Printf(" %s\n", res) } } + return errors.New("outstanding generations detected") } -func TestMain(m *testing.M) { - ret := m.Run() - examineResults() - os.Exit(ret) +func init() { + testhook.RegisterPostTestHook(examineResults) } diff --git a/generation_debug.go b/generation_debug.go index a425deabe..91c878e26 100644 --- a/generation_debug.go +++ b/generation_debug.go @@ -20,6 +20,7 @@ import ( "fmt" "math/rand" "runtime" + "runtime/debug" "sort" "sync" "time" @@ -29,6 +30,7 @@ const generationDebug = true type lifespan struct { from, to, finalized time.Time + stack []byte } var knownGenerations map[string]lifespan @@ -36,32 +38,49 @@ var knownGenerationLock sync.Mutex var timeZero time.Time +var generationDebugVerbose bool + +// History reports the finalized/dead/created status of a span which we think +// is in some way in error. It's shared between a couple of places. +func (span *lifespan) History() string { + dead := "not dead" + finalized := "not finalized" + if span.finalized != timeZero { + finalized = fmt.Sprintf("finalized at %v", span.finalized) + } + if span.to != timeZero { + dead = fmt.Sprintf("dead at %v", span.to) + } + return fmt.Sprintf("%s, %s, created at %v at %s", dead, finalized, span.from, span.stack) +} + +func (span *lifespan) reportHistory(reason string, id string) string { + return fmt.Sprintf("%s %s: %s", id, reason, span.History()) +} + func registerGeneration(id string) string { knownGenerationLock.Lock() defer knownGenerationLock.Unlock() if knownGenerations == nil { knownGenerations = make(map[string]lifespan) } - newSpan := lifespan{from: time.Now()} + newSpan := lifespan{from: time.Now(), stack: debug.Stack()} origId := id // if you have more than 65k of the same file open, maybe you have bigger // problems than this. for span, exists := knownGenerations[id]; exists; span, exists = knownGenerations[id] { suffix := fmt.Sprintf("::%04x", rand.Int63n(65536)) - if span.finalized != timeZero { - fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v, finalized %v\n", - id, suffix, span.from, span.to, span.finalized) - } else { - if span.to != timeZero { - fmt.Printf("new generation %s: adding %s, previously existed, created %v, died %v\n", id, suffix, span.from, span.to) - } else { - fmt.Printf("new generation %s: adding %s, already exists, created %v", id, suffix, span.from) - } + if generationDebugVerbose { + history := span.History() + fmt.Printf("new generation: adding suffix %s, previous %s\n", + suffix, history) } id = origId + suffix } - fmt.Printf("new generation %s\n", id) + if generationDebugVerbose { + fmt.Printf("new generation %s\n", id) + } knownGenerations[id] = newSpan return id } @@ -75,8 +94,7 @@ func endGeneration(id string) { panic(oops) } if span.finalized != timeZero || span.to != timeZero { - oops := fmt.Sprintf("ending generation %s: already died at %v, finalized at %v", id, span.to, span.finalized) - panic(oops) + panic(span.reportHistory("ending generation", id)) } span.to = time.Now() knownGenerations[id] = span @@ -108,39 +126,25 @@ func finalizeGeneration(id string) { panic(oops) } if span.finalized != timeZero { - var oops string - if span.to != timeZero { - oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, but not dead", id, span.finalized) - } else { - oops = fmt.Sprintf("finalizing generation %s: already finalized at %v, dead at %v", id, span.finalized, span.to) - } - panic(oops) + panic(span.reportHistory("finalizing", id)) } span.finalized = time.Now() knownGenerations[id] = span } -func reportGenerations() []string { +func reportGenerations() (stats string, surviving []string) { runtime.GC() knownGenerationLock.Lock() defer knownGenerationLock.Unlock() - var surviving []string times := make([]int64, 0, len(knownGenerations)) for id, span := range knownGenerations { - if span.to == timeZero { - if span.finalized == timeZero { - surviving = append(surviving, fmt.Sprintf("%s: %v, not ended or finalized", id, span.from)) - } else { - surviving = append(surviving, fmt.Sprintf("%s: %v, finalized %v, not ended", id, span.from, span.finalized)) - } + if span.to == timeZero || span.finalized == timeZero { + surviving = append(surviving, span.reportHistory("surviving", id)) } else { - if span.finalized == timeZero { - surviving = append(surviving, fmt.Sprintf("%s: %v to %v, not finalized", id, span.from, span.to)) - } else { - times = append(times, int64(span.finalized.Sub(span.to))) - } + times = append(times, int64(span.finalized.Sub(span.to))) } } + stats = "no recorded finalized spans" if len(times) > 0 { sort.Slice(times, func(i, j int) bool { return times[i] < times[j] }) var total int64 @@ -153,8 +157,8 @@ func reportGenerations() []string { p90 = times[(len(times)*9)/10] p99 = times[(len(times)*99)/100] worst = times[len(times)-1] - surviving = append(surviving, fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v", - len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst))) + stats = fmt.Sprintf("%d finalized spans. lag: mean %v, median %v, p90 %v, p99 %v, worst %v", + len(times), time.Duration(mean), time.Duration(median), time.Duration(p90), time.Duration(p99), time.Duration(worst)) } - return surviving + return stats, surviving } diff --git a/go.mod b/go.mod index 8865bb32f..2b8c8c25f 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 - github.com/glycerine/lmdb-go v1.9.26 + github.com/glycerine/lmdb-go v1.9.27 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.0 github.com/golang/protobuf v1.3.3 diff --git a/go.sum b/go.sum index 428a2e6a9..739047059 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E= github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y= -github.com/glycerine/lmdb-go v1.9.26 h1:4aIiCQhg5fLChuZuATDHD4Lr6y9CdEHLtvROkzCZKIg= -github.com/glycerine/lmdb-go v1.9.26/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs= +github.com/glycerine/lmdb-go v1.9.27 h1:k20zfiumwC/E1g/MYIzZ2GkhOH0hicDfEXa0KUjWLjY= +github.com/glycerine/lmdb-go v1.9.27/go.mod h1:DrPeeTGooMg6B7cjNSP14perptTJzzdBy5YoosthrRs= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= diff --git a/gossip/gossip.go b/gossip/gossip.go index 975c57a19..6e7ebb464 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -104,6 +104,7 @@ func (g *memberSet) Open() (err error) { // Close attempts to gracefully leave the cluster, and finally calls shutdown // after (at most) a timeout period. func (g *memberSet) Close() error { + g.eventReceiver.Close() leaveErr := g.memberlist.Leave(5 * time.Second) shutdownErr := g.memberlist.Shutdown() if leaveErr != nil || shutdownErr != nil { @@ -366,8 +367,9 @@ func (g *memberSet) MergeRemoteState(buf []byte, join bool) { // Care must be taken that events are processed in a timely manner from // the channel, since this delegate will block until an event can be sent. type eventReceiver struct { - ch chan memberlist.NodeEvent - papi *pilosa.API + ch chan memberlist.NodeEvent + closed chan struct{} + papi *pilosa.API logger logger.Logger } @@ -376,6 +378,7 @@ type eventReceiver struct { func newEventReceiver(logger logger.Logger, papi *pilosa.API) *eventReceiver { ger := &eventReceiver{ ch: make(chan memberlist.NodeEvent, 1), + closed: make(chan struct{}), logger: logger, papi: papi, } @@ -389,7 +392,10 @@ func (g *eventReceiver) NotifyJoin(n *memberlist.Node) { n2.Meta = make([]byte, len(n.Meta)) copy(n2.Meta, n.Meta) - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2} + select { + case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2}: + case <-g.closed: + } } func (g *eventReceiver) NotifyLeave(n *memberlist.Node) { @@ -398,7 +404,10 @@ func (g *eventReceiver) NotifyLeave(n *memberlist.Node) { n2.Meta = make([]byte, len(n.Meta)) copy(n2.Meta, n.Meta) - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2} + select { + case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2}: + case <-g.closed: + } } func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) { @@ -407,13 +416,25 @@ func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) { n2.Meta = make([]byte, len(n.Meta)) copy(n2.Meta, n.Meta) - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2} + select { + case g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2}: + case <-g.closed: + } +} + +func (g *eventReceiver) Close() { + close(g.closed) } func (g *eventReceiver) listen() { var nodeEventType pilosa.NodeEventType for { - e := <-g.ch + var e memberlist.NodeEvent + select { + case <-g.closed: + return + case e = <-g.ch: + } switch e.Event { case memberlist.NodeJoin: nodeEventType = pilosa.NodeJoin diff --git a/handler.go b/handler.go index 26e1bd186..619583ab2 100644 --- a/handler.go +++ b/handler.go @@ -127,7 +127,7 @@ type ImportValueRequest struct { Values []int64 // e.g. temperature, humidity, barometric pressure FloatValues []float64 StringValues []string - Clear bool // only works for ImportAtomicRecord() at the moment. + Clear bool } // AtomicRecord applies all its Ivr and Ivr atomically, in a Tx. @@ -214,7 +214,7 @@ type ImportRequest struct { RowKeys []string ColumnKeys []string Timestamps []int64 - Clear bool // only works for ImportAtomicRecord() at the moment. + Clear bool } // ValidateWithTimestamp ensures that the payload of the request is valid. diff --git a/holder.go b/holder.go index 31cd327a6..b0e312b3e 100644 --- a/holder.go +++ b/holder.go @@ -32,6 +32,7 @@ import ( "github.com/pilosa/pilosa/v2/logger" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pilosa/pilosa/v2/tracing" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" @@ -105,6 +106,8 @@ type Holder struct { opening bool Opts HolderOpts + + Auditor testhook.Auditor } type HolderOpts struct { @@ -165,7 +168,7 @@ func (lc *lockedChan) Recv() { // NewHolder returns a new instance of Holder. func NewHolder(partitionN int) *Holder { - return &Holder{ + h := &Holder{ partitionN: partitionN, indexes: make(map[string]*Index), closing: make(chan struct{}), @@ -188,7 +191,11 @@ func NewHolder(partitionN int) *Holder { Logger: logger.NopLogger, SnapshotQueue: defaultSnapshotQueue, + + Auditor: NewAuditor(), } + _ = testhook.Created(h.Auditor, h, nil) + return h } type HolderInfo struct { @@ -535,6 +542,8 @@ func (h *Holder) Open() error { err = index.Open(false) } if err != nil { + // FIXME: The holder shouldn't be responsible for closing these, probably. + _ = index.Txf.CloseDB() if err == ErrName { h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue @@ -559,6 +568,8 @@ func (h *Holder) Open() error { h.opened.Close() + _ = testhook.Opened(h.Auditor, h, nil) + return nil } @@ -627,6 +638,12 @@ func (h *Holder) Close() error { h.opened.mu.Lock() h.opened.ch = make(chan struct{}) h.opened.mu.Unlock() + if h.SnapshotQueue != nil { + h.SnapshotQueue.Stop() + h.SnapshotQueue = nil + } + + _ = testhook.Closed(h.Auditor, h, nil) return nil } diff --git a/holder_internal_test.go b/holder_internal_test.go index 008770fbc..0a9762fd4 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -16,9 +16,10 @@ package pilosa import ( "context" - "io/ioutil" "os" "testing" + + "github.com/pilosa/pilosa/v2/testhook" ) type testHolderOperator struct { @@ -72,8 +73,8 @@ func (t *testHolderOperator) ProcessFragment(*fragment) error { return nil } -func makeHolder() (*Holder, string, error) { - path, err := ioutil.TempDir("", "pilosa-") +func makeHolder(tb testing.TB) (*Holder, string, error) { + path, err := testhook.TempDir(tb, "pilosa-") if err != nil { return nil, "", err } @@ -109,7 +110,7 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui } func TestHolderOperatorProcess(t *testing.T) { - h, path, err := makeHolder() + h, path, err := makeHolder(t) if err != nil { t.Fatalf("creating holder: %v", err) } @@ -139,7 +140,7 @@ func TestHolderOperatorProcess(t *testing.T) { } func TestHolderOperatorCancel(t *testing.T) { - h, path, err := makeHolder() + h, path, err := makeHolder(t) if err != nil { t.Fatalf("creating holder: %v", err) } diff --git a/holder_test.go b/holder_test.go index b9bd4ae0a..8b3991f8c 100644 --- a/holder_test.go +++ b/holder_test.go @@ -33,7 +33,7 @@ import ( func TestHolder_Open(t *testing.T) { t.Run("ErrIndexName", func(t *testing.T) { - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) bufLogger := test.NewBufferLogger() h.Holder.Logger = bufLogger @@ -60,7 +60,7 @@ func TestHolder_Open(t *testing.T) { if os.Geteuid() == 0 { t.Skip("Skipping permissions test since user is root.") } - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { @@ -79,7 +79,7 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) { - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { @@ -99,7 +99,7 @@ func TestHolder_Open(t *testing.T) { if os.Geteuid() == 0 { t.Skip("Skipping permissions test since user is root.") } - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -119,7 +119,7 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrFieldOptionsCorrupt", func(t *testing.T) { - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() var idx *pilosa.Index @@ -142,7 +142,7 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrFieldAttrStoreCorrupt", func(t *testing.T) { - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() var idx *pilosa.Index @@ -170,7 +170,7 @@ func TestHolder_Open(t *testing.T) { if os.Geteuid() == 0 { t.Skip("Skipping permissions test since user is root.") } - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() var idx *pilosa.Index @@ -205,7 +205,7 @@ func TestHolder_Open(t *testing.T) { t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) { roaringOnlyTest(t) - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() var idx *pilosa.Index @@ -239,7 +239,7 @@ func TestHolder_Open(t *testing.T) { t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) { roaringOnlyTest(t) - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}) @@ -271,7 +271,7 @@ func TestHolder_Open(t *testing.T) { t.Run("ForeignIndex", func(t *testing.T) { t.Run("ErrForeignIndexNotFound", func(t *testing.T) { - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil { @@ -288,7 +288,7 @@ func TestHolder_Open(t *testing.T) { // Foreign index zzz is opened after foo/bar. t.Run("ForeignIndexNotOpenYet", func(t *testing.T) { - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() if _, err := h.CreateIndex("zzz", pilosa.IndexOptions{}); err != nil { @@ -308,7 +308,7 @@ func TestHolder_Open(t *testing.T) { // Foreign index aaa is opened before foo/bar. t.Run("ForeignIndexIsOpen", func(t *testing.T) { - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() if _, err := h.CreateIndex("aaa", pilosa.IndexOptions{}); err != nil { @@ -328,7 +328,7 @@ func TestHolder_Open(t *testing.T) { // Try to re-create existing index t.Run("CreateIndexIfNotExists", func(t *testing.T) { - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() idx1, err := h.CreateIndexIfNotExists("aaa", pilosa.IndexOptions{}) @@ -356,7 +356,7 @@ func TestHolder_Open(t *testing.T) { func TestHolder_HasData(t *testing.T) { t.Run("IndexDirectory", func(t *testing.T) { - h := test.MustOpenHolder() + h := test.MustOpenHolder(t) defer h.Close() if ok, err := h.HasData(); ok || err != nil { @@ -373,7 +373,7 @@ func TestHolder_HasData(t *testing.T) { }) t.Run("Peek", func(t *testing.T) { - h := test.NewHolder() + h := test.NewHolder(t) if ok, err := h.HasData(); ok || err != nil { t.Fatal("expected HasData to return false, no err, but", ok, err) @@ -390,7 +390,7 @@ func TestHolder_HasData(t *testing.T) { }) t.Run("Peek at missing directory", func(t *testing.T) { - h := test.NewHolder() + h := test.NewHolder(t) // Ensure that hasData is false when dir doesn't exist. h.Path = "bad-path" @@ -404,7 +404,7 @@ func TestHolder_HasData(t *testing.T) { // Ensure holder can delete an index and its underlying files. func TestHolder_DeleteIndex(t *testing.T) { - hldr := test.MustOpenHolder() + hldr := test.MustOpenHolder(t) defer hldr.Close() // Write bits to separate indexes. @@ -432,43 +432,43 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { c := test.MustNewCluster(t, 2) - c[0].Config.Cluster.ReplicaN = 2 - c[0].Config.AntiEntropy.Interval = 0 - c[1].Config.Cluster.ReplicaN = 2 - c[1].Config.AntiEntropy.Interval = 0 + c.GetNode(0).Config.Cluster.ReplicaN = 2 + c.GetNode(0).Config.AntiEntropy.Interval = 0 + c.GetNode(1).Config.Cluster.ReplicaN = 2 + c.GetNode(1).Config.AntiEntropy.Interval = 0 err := c.Start() if err != nil { t.Fatalf("starting cluster: %v", err) } defer c.Close() - _, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + _, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index i: %v", err) } - _, err = c[0].API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{}) + _, err = c.GetNode(0).API.CreateIndex(context.Background(), "y", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index y: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) if err != nil { t.Fatalf("creating field f: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f0", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) if err != nil { t.Fatalf("creating field f0: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "y", "z", pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) if err != nil { t.Fatalf("creating field z in y: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool()) + _, err = c.GetNode(0).API.CreateField(context.Background(), "y", "b", pilosa.OptFieldTypeBool()) if err != nil { t.Fatalf("creating field b in y: %v", err) } - hldr0 := &test.Holder{Holder: c[0].Server.Holder()} - hldr1 := &test.Holder{Holder: c[1].Server.Holder()} + hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()} + hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()} // Set data on the local holder. hldr0.SetBit("i", "f", 0, 10) @@ -495,11 +495,11 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { hldr1.SetBit("y", "b", 0, (3*ShardWidth)+5) // false hldr1.SetBit("y", "b", 1, (3*ShardWidth)+7) // true - err = c[0].Server.SyncData() + err = c.GetNode(0).Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) } - err = c[1].Server.SyncData() + err = c.GetNode(1).Server.SyncData() if err != nil { t.Fatalf("syncing node 1: %v", err) } @@ -543,30 +543,30 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // the row boundaries of the block. func TestHolderSyncer_BlockIteratorLimits(t *testing.T) { c := test.MustNewCluster(t, 3) - c[0].Config.Cluster.ReplicaN = 3 - c[0].Config.AntiEntropy.Interval = 0 - c[1].Config.Cluster.ReplicaN = 3 - c[1].Config.AntiEntropy.Interval = 0 + c.GetNode(0).Config.Cluster.ReplicaN = 3 + c.GetNode(0).Config.AntiEntropy.Interval = 0 + c.GetNode(1).Config.Cluster.ReplicaN = 3 + c.GetNode(1).Config.AntiEntropy.Interval = 0 err := c.Start() if err != nil { t.Fatalf("starting cluster: %v", err) } defer c.Close() - _, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + _, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index i: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) if err != nil { t.Fatalf("creating field f: %v", err) } blockEdge := uint64(pilosa.HashBlockSize) - hldr0 := &test.Holder{Holder: c[0].Server.Holder()} - hldr1 := &test.Holder{Holder: c[1].Server.Holder()} - hldr2 := &test.Holder{Holder: c[2].Server.Holder()} + hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()} + hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()} + hldr2 := &test.Holder{Holder: c.GetNode(2).Server.Holder()} // Set data on the local holder. hldr0.SetBit("i", "f", blockEdge-1, 10) @@ -579,7 +579,7 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) { // Leave the third replica empty to force a block merge. // - err = c[0].Server.SyncData() + err = c.GetNode(0).Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) } @@ -598,28 +598,28 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) { // Ensure holder correctly handles clears during block sync. func TestHolderSyncer_Clears(t *testing.T) { c := test.MustNewCluster(t, 3) - c[0].Config.Cluster.ReplicaN = 3 - c[0].Config.AntiEntropy.Interval = 0 - c[1].Config.Cluster.ReplicaN = 3 - c[1].Config.AntiEntropy.Interval = 0 + c.GetNode(0).Config.Cluster.ReplicaN = 3 + c.GetNode(0).Config.AntiEntropy.Interval = 0 + c.GetNode(1).Config.Cluster.ReplicaN = 3 + c.GetNode(1).Config.AntiEntropy.Interval = 0 err := c.Start() if err != nil { t.Fatalf("starting cluster: %v", err) } defer c.Close() - _, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + _, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index i: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, pilosa.DefaultCacheSize)) if err != nil { t.Fatalf("creating field f: %v", err) } - hldr0 := &test.Holder{Holder: c[0].Server.Holder()} - hldr1 := &test.Holder{Holder: c[1].Server.Holder()} - hldr2 := &test.Holder{Holder: c[2].Server.Holder()} + hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()} + hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()} + hldr2 := &test.Holder{Holder: c.GetNode(2).Server.Holder()} // Set data on the local holder that should be cleared // because it's the only instance of this value. @@ -631,7 +631,7 @@ func TestHolderSyncer_Clears(t *testing.T) { hldr1.SetBit("i", "f", 0, 20) hldr2.SetBit("i", "f", 0, 20) - err = c[0].Server.SyncData() + err = c.GetNode(0).Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) } @@ -647,10 +647,10 @@ func TestHolderSyncer_Clears(t *testing.T) { // Ensure holder can sync time quantum views with a remote holder. func TestHolderSyncer_TimeQuantum(t *testing.T) { c := test.MustNewCluster(t, 2) - c[0].Config.Cluster.ReplicaN = 2 - c[0].Config.AntiEntropy.Interval = 0 - c[1].Config.Cluster.ReplicaN = 2 - c[1].Config.AntiEntropy.Interval = 0 + c.GetNode(0).Config.Cluster.ReplicaN = 2 + c.GetNode(0).Config.AntiEntropy.Interval = 0 + c.GetNode(1).Config.Cluster.ReplicaN = 2 + c.GetNode(1).Config.AntiEntropy.Interval = 0 err := c.Start() if err != nil { t.Fatalf("starting cluster: %v", err) @@ -659,17 +659,17 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) { quantum := "D" - _, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + _, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index i: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum))) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum))) if err != nil { t.Fatalf("creating field f: %v", err) } - hldr0 := &test.Holder{Holder: c[0].Server.Holder()} - hldr1 := &test.Holder{Holder: c[1].Server.Holder()} + hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()} + hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()} // Set data on the local holder for node0. t1 := time.Date(2018, 8, 1, 12, 30, 0, 0, time.UTC) @@ -680,7 +680,7 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) { // Set data on node1. hldr1.SetBitTime("i", "f", 0, 22, &t2) - err = c[0].Server.SyncData() + err = c.GetNode(0).Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) } @@ -700,10 +700,10 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) { func TestHolderSyncer_IntField(t *testing.T) { t.Run("BasicSync", func(t *testing.T) { c := test.MustNewCluster(t, 2) - c[0].Config.Cluster.ReplicaN = 2 - c[0].Config.AntiEntropy.Interval = 0 - c[1].Config.Cluster.ReplicaN = 2 - c[1].Config.AntiEntropy.Interval = 0 + c.GetNode(0).Config.Cluster.ReplicaN = 2 + c.GetNode(0).Config.AntiEntropy.Interval = 0 + c.GetNode(1).Config.Cluster.ReplicaN = 2 + c.GetNode(1).Config.AntiEntropy.Interval = 0 err := c.Start() if err != nil { t.Fatalf("starting cluster: %v", err) @@ -712,18 +712,18 @@ func TestHolderSyncer_IntField(t *testing.T) { var idx0 *pilosa.Index _ = idx0 - idx0, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) _ = idx0 if err != nil { t.Fatalf("creating index i: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(0, 100)) if err != nil { t.Fatalf("creating field f: %v", err) } - hldr0 := &test.Holder{Holder: c[0].Server.Holder()} - hldr1 := &test.Holder{Holder: c[1].Server.Holder()} + hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()} + hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()} // Set data on the local holder for node0. columnID=1, value=1 hldr0.SetValue("i", "f", 1, 1) @@ -734,7 +734,7 @@ func TestHolderSyncer_IntField(t *testing.T) { idx1 := hldr1.SetValue("i", "f", 2, 2) _ = idx1 - err = c[0].Server.SyncData() + err = c.GetNode(0).Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) } @@ -758,10 +758,10 @@ func TestHolderSyncer_IntField(t *testing.T) { t.Run("MultiShard", func(t *testing.T) { c := test.MustNewCluster(t, 2) - c[0].Config.Cluster.ReplicaN = 2 - c[0].Config.AntiEntropy.Interval = 0 - c[1].Config.Cluster.ReplicaN = 2 - c[1].Config.AntiEntropy.Interval = 0 + c.GetNode(0).Config.Cluster.ReplicaN = 2 + c.GetNode(0).Config.AntiEntropy.Interval = 0 + c.GetNode(1).Config.Cluster.ReplicaN = 2 + c.GetNode(1).Config.AntiEntropy.Interval = 0 err := c.Start() if err != nil { t.Fatalf("starting cluster: %v", err) @@ -770,18 +770,18 @@ func TestHolderSyncer_IntField(t *testing.T) { var idx0 *pilosa.Index _ = idx0 - idx0, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + idx0, err = c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) _ = idx0 if err != nil { t.Fatalf("creating index i: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) if err != nil { t.Fatalf("creating field f: %v", err) } - hldr0 := &test.Holder{Holder: c[0].Server.Holder()} - hldr1 := &test.Holder{Holder: c[1].Server.Holder()} + hldr0 := &test.Holder{Holder: c.GetNode(0).Server.Holder()} + hldr1 := &test.Holder{Holder: c.GetNode(1).Server.Holder()} // Set data on the local holder for node0. hldr0.SetValue("i", "f", 1*pilosa.ShardWidth, 11) @@ -799,11 +799,11 @@ func TestHolderSyncer_IntField(t *testing.T) { // node0: [0,3,7] // node1: [1,2,4] - err = c[0].Server.SyncData() + err = c.GetNode(0).Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) } - err = c[1].Server.SyncData() + err = c.GetNode(1).Server.SyncData() if err != nil { t.Fatalf("syncing node 1: %v", err) } diff --git a/http/client_test.go b/http/client_test.go index ff543def5..5e4b0c7db 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -49,7 +49,7 @@ func TestClient_MultiNode(t *testing.T) { defer c.Close() hldr := []test.Holder{} - for _, command := range c { + for _, command := range c.Nodes { hldr = append(hldr, test.Holder{Holder: command.Server.Holder()}) } @@ -72,7 +72,7 @@ func TestClient_MultiNode(t *testing.T) { } } if !ownsNum { - t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c[i].URL(), owns) + t.Fatalf("Trying to use shard %d on host %s, but it doesn't own that shard. It owns %v", num, c.GetNode(i).URL(), owns) } } @@ -86,11 +86,11 @@ func TestClient_MultiNode(t *testing.T) { maxShard = x } } - _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + _, err := c.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) if err != nil { t.Fatalf("creating field: %v", err) } @@ -119,24 +119,24 @@ func TestClient_MultiNode(t *testing.T) { // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay // built into cache.Invalidate() - err = c[0].RecalculateCaches(t) + err = c.GetNode(0).RecalculateCaches(t) if err != nil { t.Fatalf("recalculating cache: %v", err) } - err = c[1].RecalculateCaches(t) + err = c.GetNode(1).RecalculateCaches(t) if err != nil { t.Fatalf("recalculating cache: %v", err) } - err = c[2].RecalculateCaches(t) + err = c.GetNode(2).RecalculateCaches(t) if err != nil { t.Fatalf("recalculating cache: %v", err) } // Connect to each node to compare results. client := make([]*Client, 3) - client[0] = MustNewClient(c[0].URL(), http.GetHTTPClient(nil)) - client[1] = MustNewClient(c[1].URL(), http.GetHTTPClient(nil)) - client[2] = MustNewClient(c[2].URL(), http.GetHTTPClient(nil)) + client[0] = MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil)) + client[1] = MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil)) + client[2] = MustNewClient(c.GetNode(2).URL(), http.GetHTTPClient(nil)) topN := 4 queryRequest := &pilosa.QueryRequest{ @@ -188,7 +188,7 @@ func TestClient_MultiNode(t *testing.T) { func TestClient_Export(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) host := cmd.URL() @@ -368,7 +368,7 @@ func TestClient_Export(t *testing.T) { func TestClient_Import(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -415,7 +415,7 @@ func TestClient_Import(t *testing.T) { // Ensure client can bulk import column attrs. func TestClient_ImportColumnAttrs(t *testing.T) { cluster := test.MustNewCluster(t, 2) - for _, c := range cluster { + for _, c := range cluster.Nodes { c.Config.Cluster.ReplicaN = 2 } err := cluster.Start() @@ -425,31 +425,31 @@ func TestClient_ImportColumnAttrs(t *testing.T) { defer cluster.Close() ctx := context.Background() - _, err = cluster[0].API.CreateIndex(ctx, "i", pilosa.IndexOptions{}) + _, err = cluster.GetNode(0).API.CreateIndex(ctx, "i", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = cluster[0].API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + _, err = cluster.GetNode(0).API.CreateField(ctx, "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) if err != nil { t.Fatalf("creating field: %v", err) } - _, err = cluster[0].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"}) + _, err = cluster.GetNode(0).API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=0) Set(1, f=0) Set(2, f=0) Set(3, f=0) Set(4, f=0)"}) if err != nil { t.Fatalf("querying: %v", err) } attrKey := "k" // Send import request. - host := cluster[0].URL() + host := cluster.GetNode(0).URL() c := MustNewClient(host, http.GetHTTPClient(nil)) colAttrsReq := makeImportColumnAttrsRequest("i", 0, attrKey) - if err := c.ImportColumnAttrs(ctx, &cluster[1].API.Node().URI, "i", colAttrsReq); err != nil { + if err := c.ImportColumnAttrs(ctx, &cluster.GetNode(1).API.Node().URI, "i", colAttrsReq); err != nil { t.Fatal(err) } // Verify data. pql := "Options(Row(f=0), columnAttrs=true)" - res, err := cluster[1].API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql}) + res, err := cluster.GetNode(1).API.Query(ctx, &pilosa.QueryRequest{Index: "i", Query: pql}) if err != nil { t.Fatal(err) } @@ -469,7 +469,7 @@ func TestClient_ImportColumnAttrs(t *testing.T) { // Ensure client can bulk import data. func TestClient_ImportRoaring(t *testing.T) { cluster := test.MustNewCluster(t, 2) - for _, c := range cluster { + for _, c := range cluster.Nodes { c.Config.Cluster.ReplicaN = 2 } err := cluster.Start() @@ -478,29 +478,29 @@ func TestClient_ImportRoaring(t *testing.T) { } defer cluster.Close() - _, err = cluster[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + _, err = cluster.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) if err != nil { t.Fatalf("creating index: %v", err) } - _, err = cluster[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + _, err = cluster.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) if err != nil { t.Fatalf("creating field: %v", err) } - _, err = cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"}) + _, err = cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"}) if err != nil { t.Fatalf("querying: %v", err) } // Send import request. - host := cluster[0].URL() + host := cluster.GetNode(0).URL() c := MustNewClient(host, http.GetHTTPClient(nil)) // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } - hldr := test.Holder{Holder: cluster[0].Server.Holder()} + hldr := test.Holder{Holder: cluster.GetNode(0).Server.Holder()} // Verify data on node 0. if a := hldr.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) { t.Fatalf("unexpected columns: %+v", a) @@ -509,7 +509,7 @@ func TestClient_ImportRoaring(t *testing.T) { t.Fatalf("unexpected columns: %+v", a) } - hldr2 := test.Holder{Holder: cluster[1].Server.Holder()} + hldr2 := test.Holder{Holder: cluster.GetNode(1).Server.Holder()} // Verify data on node 1. if a := hldr2.Row("i", "f", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537}) { t.Fatalf("unexpected columns: %+v", a) @@ -521,7 +521,7 @@ func TestClient_ImportRoaring(t *testing.T) { // Ensure that sending a roaring import with the clear flag works as expected. // [65539, 65540] roaringReq = makeImportRoaringRequest(true, "3A30000001000000010001001000000003000400") - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -544,7 +544,7 @@ func TestClient_ImportRoaring(t *testing.T) { // Ensure that sending a roaring import with the clear flag works as expected. // [4, 6, 65537, 65539] roaringReq = makeImportRoaringRequest(true, "3A300000020000000000010001000100180000001C0000000400060001000300") - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -567,7 +567,7 @@ func TestClient_ImportRoaring(t *testing.T) { // Ensure that sending a roaring import with the clear flag works as expected. // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] roaringReq = makeImportRoaringRequest(true, "3B3001000100000900010000000100010009000100") - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -593,7 +593,7 @@ func TestClient_ImportKeys(t *testing.T) { t.Run("SingleNode", func(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) host := cmd.URL() cmd.MustCreateIndex(t, "keyed", pilosa.IndexOptions{Keys: true}) @@ -691,8 +691,8 @@ func TestClient_ImportKeys(t *testing.T) { t.Run("MultiNode", func(t *testing.T) { cluster := test.MustRunCluster(t, 2) defer cluster.Close() - cmd0 := cluster[0] - cmd1 := cluster[1] + cmd0 := cluster.GetNode(0) + cmd1 := cluster.GetNode(1) host0 := cmd0.URL() host1 := cmd1.URL() @@ -768,7 +768,7 @@ func TestClient_ImportKeys(t *testing.T) { t.Run("IntegerFieldSingleNode", func(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -840,7 +840,7 @@ func TestClient_ImportIDs(t *testing.T) { t.Run("ImportRangeImport", func(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -901,7 +901,7 @@ func TestClient_ImportIDs(t *testing.T) { func TestClient_ImportValue(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -974,7 +974,7 @@ func TestClient_ImportValue(t *testing.T) { func TestClient_ImportExistence(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) host := cmd.URL() holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -1053,7 +1053,7 @@ func TestClient_ImportExistence(t *testing.T) { func TestClient_FragmentBlocks(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -1086,7 +1086,7 @@ func TestClient_FragmentBlocks(t *testing.T) { func TestClient_CreateDecimalField(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) c := MustNewClient(cmd.URL(), http.GetHTTPClient(nil)) @@ -1195,8 +1195,8 @@ func TestClientTransactions(t *testing.T) { c := test.MustRunCluster(t, 3) defer c.Close() - client0 := MustNewClient(c[0].URL(), http.GetHTTPClient(nil)) - client1 := MustNewClient(c[1].URL(), http.GetHTTPClient(nil)) + client0 := MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil)) + client1 := MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil)) // can create, list, get, and finish a transaction var expDeadline time.Time diff --git a/http/translator_test.go b/http/translator_test.go index 8d4a9362d..c6c9a0519 100644 --- a/http/translator_test.go +++ b/http/translator_test.go @@ -36,7 +36,7 @@ func TestTranslateStore_EntryReader(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - primary := cluster[0] + primary := cluster.GetNode(0) hldr := test.Holder{Holder: primary.Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) @@ -157,7 +157,7 @@ func benchmarkSetup(b *testing.B, ctx context.Context, key string, nkeys int) (s b.Helper() cluster := test.MustRunCluster(b, 1) - primary := cluster[0] + primary := cluster.GetNode(0) idx := primary.MustCreateIndex(b, "i", pilosa.IndexOptions{}) fld := primary.MustCreateField(b, idx.Name(), "f", pilosa.OptFieldKeys()) diff --git a/index.go b/index.go index 8b60f2481..354615ea3 100644 --- a/index.go +++ b/index.go @@ -29,6 +29,7 @@ import ( "github.com/pilosa/pilosa/v2/internal" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -153,6 +154,9 @@ func (i *Index) CreatedAt() int64 { // Name returns name of the index. func (i *Index) Name() string { return i.name } +// Holder yields this index's Holder. +func (i *Index) Holder() *Holder { return i.holder } + // QualifiedName returns the qualified name of the index. func (i *Index) QualifiedName() string { return i.qualifiedName } @@ -253,6 +257,7 @@ func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) { return err } + _ = testhook.Opened(i.holder.Auditor, i, nil) return nil } @@ -339,7 +344,16 @@ fileLoop: }) } } - return eg.Wait() + err = eg.Wait() + if err != nil { + // Close any fields which got opened, since the overall + // index won't be open. + for n, f := range i.fields { + f.Close() + delete(i.fields, n) + } + } + return err } // openExistenceField gets or creates the existence field and associates it to the index. @@ -403,6 +417,9 @@ func (i *Index) saveMeta() error { func (i *Index) Close() error { i.mu.Lock() defer i.mu.Unlock() + defer func() { + _ = testhook.Closed(i.holder.Auditor, i, nil) + }() err := i.Txf.CloseIndex(i) if err != nil { diff --git a/index_internal_test.go b/index_internal_test.go index 4f569c745..4981ed191 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -15,19 +15,23 @@ package pilosa import ( - "io/ioutil" "testing" + + "github.com/pilosa/pilosa/v2/testhook" ) // mustOpenIndex returns a new, opened index at a temporary path. Panic on error. -func mustOpenIndex(opt IndexOptions) *Index { - path, err := ioutil.TempDir(*TempDir, "pilosa-index-") +func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index { + path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-index-") if err != nil { panic(err) } h := NewHolder(1) h.Path = path index, err := h.CreateIndex("i", opt) + testhook.Cleanup(tb, func() { + h.Close() + }) if err != nil { panic(err) @@ -36,9 +40,6 @@ func mustOpenIndex(opt IndexOptions) *Index { index.keys = opt.Keys index.trackExistence = opt.TrackExistence - if err := index.Open(false); err != nil { - panic(err) - } return index } @@ -56,7 +57,7 @@ func (i *Index) reopen() error { // Ensure that deleting the existence field is handled properly. func TestIndex_Existence_Delete(t *testing.T) { // Create Index (with existence tracking). - index := mustOpenIndex(IndexOptions{TrackExistence: true}) + index := mustOpenIndex(t, IndexOptions{TrackExistence: true}) defer index.Close() // Ensure existence field has been created. diff --git a/index_test.go b/index_test.go index bfebee7f7..ded04b164 100644 --- a/index_test.go +++ b/index_test.go @@ -15,13 +15,13 @@ package pilosa_test import ( - "io/ioutil" "reflect" "testing" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/test" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" ) @@ -30,7 +30,7 @@ const ShardWidth = pilosa.ShardWidth // Ensure index can open and retrieve a field. func TestIndex_CreateFieldIfNotExists(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() // Create field. @@ -58,7 +58,7 @@ func TestIndex_CreateField(t *testing.T) { // Ensure time quantum can be set appropriately on a new field. t.Run("TimeQuantum", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() // Create field with explicit quantum. @@ -74,7 +74,7 @@ func TestIndex_CreateField(t *testing.T) { // Ensure time quantum can be set appropriately on a new field. t.Run("TimeQuantumNoStandardView", func(t *testing.T) { t.Run("Explicit", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() // Create field with explicit quantum with no standard view @@ -90,7 +90,7 @@ func TestIndex_CreateField(t *testing.T) { // Ensure field can include range columns. t.Run("BSIFields", func(t *testing.T) { t.Run("OK", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() // Create field with schema and verify it exists. @@ -112,7 +112,7 @@ func TestIndex_CreateField(t *testing.T) { // on field creation FieldOptions validation. /* t.Run("ErrRangeCacheAllowed", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() if _, err := index.CreateField("f", pilosa.FieldOptions{ @@ -123,7 +123,7 @@ func TestIndex_CreateField(t *testing.T) { }) t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() if _, err := index.CreateField("f", pilosa.FieldOptions{ CacheType: pilosa.CacheTypeNone, @@ -134,7 +134,7 @@ func TestIndex_CreateField(t *testing.T) { }) t.Run("ErrFieldFieldsAllowed", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() if _, err := index.CreateField("f", pilosa.FieldOptions{ @@ -147,7 +147,7 @@ func TestIndex_CreateField(t *testing.T) { }) t.Run("ErrFieldNameRequired", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() if _, err := index.CreateField("f", pilosa.FieldOptions{ @@ -160,7 +160,7 @@ func TestIndex_CreateField(t *testing.T) { }) t.Run("ErrInvalidFieldType", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() if _, err := index.CreateField("f", pilosa.FieldOptions{ @@ -173,7 +173,7 @@ func TestIndex_CreateField(t *testing.T) { }) t.Run("ErrInvalidBSIGroupRange", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() if _, err := index.CreateField("f", pilosa.FieldOptions{ @@ -190,7 +190,7 @@ func TestIndex_CreateField(t *testing.T) { t.Run("WithKeys", func(t *testing.T) { // Don't allow an int field to be created with keys=true t.Run("IntField", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() _, err := index.CreateField("f", pilosa.OptFieldTypeInt(-1, 1), pilosa.OptFieldKeys()) @@ -201,7 +201,7 @@ func TestIndex_CreateField(t *testing.T) { // Don't allow a decimal field to be created with keys=true t.Run("DecimalField", func(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() _, err := index.CreateField("f", pilosa.OptFieldTypeDecimal(1, pql.Decimal{Value: -1}, pql.Decimal{Value: 1}), pilosa.OptFieldKeys()) @@ -214,7 +214,7 @@ func TestIndex_CreateField(t *testing.T) { // Ensure index can delete a field. func TestIndex_DeleteField(t *testing.T) { - index := test.MustOpenIndex() + index := test.MustOpenIndex(t) defer index.Close() // Create field. @@ -238,7 +238,7 @@ func TestIndex_DeleteField(t *testing.T) { // Ensure index can validate its name. func TestIndex_InvalidName(t *testing.T) { - path, err := ioutil.TempDir("", "pilosa-index-") + path, err := testhook.TempDir(t, "pilosa-index-") if err != nil { panic(err) } diff --git a/lmdb.go b/lmdb.go index b67436abf..b928b01da 100644 --- a/lmdb.go +++ b/lmdb.go @@ -22,9 +22,11 @@ import ( "io" "io/ioutil" "log" + "math" "os" "path/filepath" "runtime" + "sort" "strings" "sync" "sync/atomic" @@ -128,14 +130,15 @@ func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) { err = env.SetMaxDBs(1) panicOn(err) - err = env.SetMapSize(256 << 30) // 256GB + //err = env.SetMapSize(256 << 30) // 256GB + err = env.SetMapSize(16 << 30) // 16GB panicOn(err) panicOn(os.MkdirAll(filepath.Dir(path), 0755)) flags := uint(lmdb.NoReadahead | lmdb.NoSubdir) - // unsafe, but get upper bound on performance. TODO: remove these. + // unsafe, but get upper bound on performance. // WriteMap = C.MDB_WRITEMAP // Use a writable memory map. // NoMetaSync = C.MDB_NOMETASYNC // Don't fsync metapage after commit. // NoSync = C.MDB_NOSYNC // Don't fsync after commit. @@ -145,12 +148,20 @@ func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) { // kRemove N= 710401 avg/op: 7.714µs sd: 27.83µs total: 5.480656859s // kAdd N= 722835 avg/op: 9.096µs sd: 105.787µs total: 6.575497725s + // ACI not ACID at the moment; no durability flags = flags | + lmdb.NoMemInit | // Disable LMDB memory initialization + + // Note that lmdb.WriteMap requests a big, writable, memory map. + // On my darwin/OSX laptop with 16GB ram, for instance, we + // can have difficulty obtaining this, resulting in + // panic: mdb_env_open: no space left on device lmdb.WriteMap | // Use a writable memory map. - //lmdb.NoMetaSync | // Don't fsync metapage after commit. - //lmdb.NoSync | // Don't fsync after commit. - //lmdb.MapAsync | // Flush asynchronously when using the WriteMap flag. - lmdb.NoMemInit // Disable LMDB memory initialization + + // default ACI (not Durable) transactions; 300% faster write speed results. + lmdb.NoMetaSync | // Don't fsync metapage after commit. + lmdb.NoSync | // Don't fsync after commit. + lmdb.MapAsync // Flush asynchronously when using the WriteMap flag. err = env.Open(path, flags, 0644) if err != nil { @@ -476,80 +487,86 @@ func (tx *LMDBTx) RemoveContainer(index, field, view string, shard uint64, ckey // Add sets all the a bits hot in the specified fragment. func (tx *LMDBTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { + return tx.addOrRemove(index, field, view, shard, batched, false, a...) +} +// Remove clears all the specified a bits in the chosen fragment. +func (tx *LMDBTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { + const batched = false + const remove = true + return tx.addOrRemove(index, field, view, shard, batched, remove, a...) +} + +func (tx *LMDBTx) addOrRemove(index, field, view string, shard uint64, batched, remove bool, a ...uint64) (changeCount int, err error) { // pure hack to match RoaringTx defer func() { - if !batched { + if !remove && !batched { if changeCount > 0 { changeCount = 1 } } }() - // TODO: optimization: group 'a' elements into their containers, - // and then do all the Adds on that - // container at once, so we don't retrieve a container per bit. - // (maybe, for example, using ImportRoaringBits with clear=false). - - for _, v := range a { - hi, lo := highbits(v), lowbits(v) - - var rct *roaring.Container - rct, err = tx.Container(index, field, view, shard, hi) - panicOn(err) - if err != nil { - return 0, err - } - chng := false - // TODO optimization: set all the bits in the current container at once. group by container first. - rc1, chng := rct.Add(lo) - panicOn(err) - if chng { - changeCount++ - } - if err != nil { - return changeCount, err - } - err = tx.PutContainer(index, field, view, shard, hi, rc1) - //panicOn(err) + if len(a) == 0 { + return 0, nil } - return -} -// Remove clears all the specified a bits in the chosen fragment. -func (tx *LMDBTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { + // have to sort, b/c input is not always sorted. + sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) - // TODO: optimization: group 'a' elements into their containers, - // and then do all the Removes on that - // container at once, so we don't retrieve a container per bit. - // (maybe, for example, using ImportRoaringBits with clear=true). - for _, v := range a { - hi, lo := highbits(v), lowbits(v) + var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. + var rc *roaring.Container + var hi uint64 + var lo uint16 - var rct *roaring.Container - rct, err = tx.Container(index, field, view, shard, hi) - panicOn(err) - if err != nil { - return 0, err - } + for i, v := range a { + + hi, lo = highbits(v), lowbits(v) + if hi != lastHi { + // either first time through, or changed to a different container. + // do we need put the last updated container now? + if i > 0 { + // not first time through, write what we got. + if remove && (rc == nil || rc.N() == 0) { + err = tx.RemoveContainer(index, field, view, shard, lastHi) + panicOn(err) + } else { + err = tx.PutContainer(index, field, view, shard, lastHi, rc) + panicOn(err) + } + } + // get the next container + rc, err = tx.Container(index, field, view, shard, hi) + panicOn(err) + } // else same container, keep adding bits to rct. chng := false - rc1, chng := rct.Remove(lo) - panicOn(err) + // rc can be nil before, and nil after, in both Remove/Add below. + // The roaring container add() and remove() methods handle this. + if remove { + rc, chng = rc.Remove(lo) + } else { + rc, chng = rc.Add(lo) + } if chng { changeCount++ } - if err != nil { - return changeCount, err - } - if rc1.N() == 0 { + lastHi = hi + } + // write the last updates. + if remove { + if rc == nil || rc.N() == 0 { err = tx.RemoveContainer(index, field, view, shard, hi) - if err != nil { - return - } + panicOn(err) } else { - err = tx.PutContainer(index, field, view, shard, hi, rc1) + err = tx.PutContainer(index, field, view, shard, hi, rc) panicOn(err) } + } else { + if rc == nil || rc.N() == 0 { + panic("there should be no way to have an empty bitmap AFTER an Add() operation") + } + err = tx.PutContainer(index, field, view, shard, hi, rc) + panicOn(err) } return } diff --git a/main_test.go b/main_test.go new file mode 100644 index 000000000..cf4da6a7c --- /dev/null +++ b/main_test.go @@ -0,0 +1,25 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa_test + +import ( + "testing" + + "github.com/pilosa/pilosa/v2/testhook" +) + +func TestMain(m *testing.M) { + testhook.RunTestsWithHooks(m) +} diff --git a/mmap_test.go b/mmap_test.go index 28f00c81a..7131118d1 100644 --- a/mmap_test.go +++ b/mmap_test.go @@ -31,11 +31,12 @@ type cv struct { func forceSnapshotsCheckMapping(t *testing.T) { depth := uint(6) - f, idx := mustOpenBSIFragment("i", "f", viewStandard, 0) + f, idx, tx := mustOpenBSIFragment(t, "i", "f", viewStandard, 0) + tx.Rollback() f.Logger = logger.NewLogfLogger(t) defer f.Clean(t) - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) + tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) defer tx.Rollback() for i := 0; i < f.MaxOpN; i++ { diff --git a/pg/server_test.go b/pg/server_test.go index 9eda44bcf..c2f2e897e 100644 --- a/pg/server_test.go +++ b/pg/server_test.go @@ -35,7 +35,7 @@ func TestStartupTimeout(t *testing.T) { connect, shutdown, err := pgtest.ServeMem(&pg.Server{ StartupTimeout: time.Millisecond, - Logger: logger.NewLogfLogger(t), + Logger: logger.NopLogger, }) if err != nil { t.Fatalf("starting in-memory postgres server: %v", err) @@ -103,7 +103,7 @@ func TestPQConnect(t *testing.T) { server := &pg.Server{ StartupTimeout: time.Second, - Logger: logger.NewLogfLogger(t), + Logger: logger.NopLogger, } addr, shutdown, err := pgtest.ServeTCP(":0", server) if err != nil { @@ -134,7 +134,7 @@ func TestPQConnectSSL(t *testing.T) { server := &pg.Server{ StartupTimeout: time.Second, - Logger: logger.NewLogfLogger(t), + Logger: logger.NopLogger, } addr, shutdown, err := pgtest.ServeTLS(":0", server) if err != nil { @@ -197,7 +197,7 @@ func TestPSQLQuery(t *testing.T) { }), TypeEngine: pg.PrimitiveTypeEngine{}, StartupTimeout: time.Second, - Logger: logger.NewLogfLogger(t), + Logger: logger.NopLogger, } addr, shutdown, err := pgtest.ServeTCP(":0", server) if err != nil { diff --git a/rbf.go b/rbf.go index 31326f8ea..a7723e77d 100644 --- a/rbf.go +++ b/rbf.go @@ -19,7 +19,9 @@ import ( "fmt" "io" "io/ioutil" + "math" "os" + "sort" "strings" "sync" @@ -164,12 +166,90 @@ func (tx *RBFTx) RemoveContainer(index, field, view string, shard uint64, key ui return tx.tx.RemoveContainer(rbfName(index, field, view, shard), key) } +// Add sets all the a bits hot in the specified fragment. func (tx *RBFTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { - return tx.tx.Add(rbfName(index, field, view, shard), a...) + return tx.addOrRemove(index, field, view, shard, batched, false, a...) } +// Remove clears all the specified a bits in the chosen fragment. func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - return tx.tx.Remove(rbfName(index, field, view, shard), a...) + const batched = false + const remove = true + return tx.addOrRemove(index, field, view, shard, batched, remove, a...) +} + +func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, batched, remove bool, a ...uint64) (changeCount int, err error) { + // pure hack to match RoaringTx + defer func() { + if !remove && !batched { + if changeCount > 0 { + changeCount = 1 + } + } + }() + + if len(a) == 0 { + return 0, nil + } + + // have to sort, b/c input is not always sorted. + sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) + + var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. + var rc *roaring.Container + var hi uint64 + var lo uint16 + + for i, v := range a { + + hi, lo = highbits(v), lowbits(v) + if hi != lastHi { + // either first time through, or changed to a different container. + // do we need put the last updated container now? + if i > 0 { + // not first time through, write what we got. + if remove && (rc == nil || rc.N() == 0) { + err = tx.RemoveContainer(index, field, view, shard, lastHi) + panicOn(err) + } else { + err = tx.PutContainer(index, field, view, shard, lastHi, rc) + panicOn(err) + } + } + // get the next container + rc, err = tx.Container(index, field, view, shard, hi) + panicOn(err) + } // else same container, keep adding bits to rct. + chng := false + // rc can be nil before, and nil after, in both Remove/Add below. + // The roaring container add() and remove() methods handle this. + if remove { + rc, chng = rc.Remove(lo) + } else { + rc, chng = rc.Add(lo) + } + if chng { + changeCount++ + } + lastHi = hi + } + // write the last updates. + if remove { + if rc == nil || rc.N() == 0 { + err = tx.RemoveContainer(index, field, view, shard, hi) + panicOn(err) + } else { + err = tx.PutContainer(index, field, view, shard, hi, rc) + panicOn(err) + } + } else { + if rc == nil || rc.N() == 0 { + panic("there should be no way to have an empty bitmap AFTER an Add() operation") + } + err = tx.PutContainer(index, field, view, shard, hi, rc) + panicOn(err) + } + return } func (tx *RBFTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) { diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 32704988e..b9bc99373 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -94,7 +94,7 @@ func TestTx_CommitRollback(t *testing.T) { }) t.Run("SingleWriter", func(t *testing.T) { - t.Skip("NEED TO FIX IN RACE") //TODO (twg) + //t.Skip("NEED TO FIX IN RACE") //TODO (twg) db := MustOpenDB(t) defer MustCloseDB(t, db) diff --git a/rrtx.go b/rrtx.go index a9bfe647c..894d2e521 100644 --- a/rrtx.go +++ b/rrtx.go @@ -369,6 +369,7 @@ func (db *RoaringStore) DeleteField(index, field, fieldPath string) error { } // frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil. +// The fragment should be closed before this. func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { fragment, ok := frag.(*fragment) @@ -376,11 +377,6 @@ func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag) } - // Close data files before deletion. - if err := fragment.Close(); err != nil { - return errors.Wrap(err, "closing fragment") - } - // Delete fragment file. if err := os.Remove(fragment.path); err != nil { return errors.Wrap(err, "deleting fragment file") diff --git a/server/cluster_test.go b/server/cluster_test.go index eb281d08f..b3050aaf2 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -35,7 +35,7 @@ import ( // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { ms := test.MustRunCluster(t, 2) - m0, m1 := ms[0], ms[1] + m0, m1 := ms.GetNode(0), ms.GetNode(1) defer ms.Close() // Expected indexes and Fields @@ -131,10 +131,10 @@ func TestClusterResize_EmptyNodes(t *testing.T) { clus := test.MustRunCluster(t, 2) defer clus.Close() - if clus[0].API.State() != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) - } else if clus[1].API.State() != pilosa.ClusterStateNormal { - t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State()) + if clus.GetNode(0).API.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", clus.GetNode(0).API.State()) + } else if clus.GetNode(1).API.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", clus.GetNode(1).API.State()) } } @@ -144,15 +144,15 @@ func TestClusterResize_AddNode(t *testing.T) { clus := test.MustRunCluster(t, 2) defer clus.Close() - if !test.CheckClusterState(clus[0], pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) - } else if !test.CheckClusterState(clus[1], pilosa.ClusterStateNormal, 1000) { - t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State()) + if !test.CheckClusterState(clus.GetNode(0), pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", clus.GetNode(0).API.State()) + } else if !test.CheckClusterState(clus.GetNode(1), pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", clus.GetNode(1).API.State()) } }) t.Run("WithIndex", func(t *testing.T) { // Configure node0 - m0 := test.MustRunCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() seed := m0.GossipAddress() @@ -168,7 +168,7 @@ func TestClusterResize_AddNode(t *testing.T) { } // Configure node1 - m1 := test.NewCommandNode(false) + m1 := test.NewCommandNode(t, false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} err := m1.Start() @@ -198,7 +198,7 @@ func TestClusterResize_AddNode(t *testing.T) { skipTestUnderBlueGreenWithRoaring(t) // Configure node0 - m0 := test.MustRunCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() seed := m0.GossipAddress() @@ -229,7 +229,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0.QueryExpect(t, "i", "", `Row(f=1)`, exp) // Configure node1 - m1 := test.NewCommandNode(false) + m1 := test.NewCommandNode(t, false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} err := m1.Start() @@ -250,7 +250,7 @@ func TestClusterResize_AddNode(t *testing.T) { }) t.Run("OneShard", func(t *testing.T) { // Configure node0 - m0 := test.MustRunCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() seed := m0.GossipAddress() @@ -278,7 +278,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0.QueryExpect(t, "i", "", `Row(f=1)`, exp) // Configure node1 - m1 := test.NewCommandNode(false) + m1 := test.NewCommandNode(t, false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} err := m1.Start() @@ -299,7 +299,7 @@ func TestClusterResize_AddNode(t *testing.T) { }) t.Run("SkippedShard", func(t *testing.T) { // Configure node0 - m0 := test.MustRunCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() seed := m0.GossipAddress() @@ -331,7 +331,7 @@ func TestClusterResize_AddNode(t *testing.T) { m0.QueryExpect(t, "i", "", `Row(f=1)`, exp) // Configure node1 - m1 := test.NewCommandNode(false) + m1 := test.NewCommandNode(t, false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} err := m1.Start() @@ -356,7 +356,7 @@ func TestClusterResize_AddNode(t *testing.T) { func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { t.Run("WithIndex", func(t *testing.T) { // Configure node0 - m0 := test.MustRunCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() seed := m0.GossipAddress() @@ -378,7 +378,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { }() // Configure node1 - m1 := test.NewCommandNode(false) + m1 := test.NewCommandNode(t, false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} err := m1.Start() @@ -399,7 +399,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { }) t.Run("ContinuousShards", func(t *testing.T) { // Configure node0 - m0 := test.MustRunCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() seed := m0.GossipAddress() @@ -431,7 +431,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { m0.QueryExpect(t, "i", "", `Row(f=1)`, exp) // Configure node1 - m1 := test.NewCommandNode(false) + m1 := test.NewCommandNode(t, false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} err := m1.Start() @@ -457,7 +457,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { }) t.Run("SkippedShard", func(t *testing.T) { // Configure node0 - m0 := test.MustRunCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() seed := m0.GossipAddress() @@ -489,7 +489,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { m0.QueryExpect(t, "i", "", `Row(f=1)`, exp) // Configure node1 - m1 := test.NewCommandNode(false) + m1 := test.NewCommandNode(t, false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} errc := make(chan error, 1) @@ -515,7 +515,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { }) t.Run("WithIndexKeys", func(t *testing.T) { // Configure node0 - m0 := test.MustRunCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() seed := m0.GossipAddress() @@ -545,7 +545,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { m0.QueryExpect(t, "i", "", `Row(f=1)`, exp) // Configure node1 - m1 := test.NewCommandNode(false) + m1 := test.NewCommandNode(t, false) m1.Config.Gossip.Port = "0" m1.Config.Gossip.Seeds = []string{seed} errc := make(chan error, 1) @@ -573,7 +573,7 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { func TestCluster_GossipMembership(t *testing.T) { t.Run("Node0Down", func(t *testing.T) { // Configure node0 - m0 := test.MustRunCluster(t, 1)[0] + m0 := test.MustRunCluster(t, 1).GetNode(0) defer m0.Close() seed := m0.GossipAddress() @@ -581,7 +581,7 @@ func TestCluster_GossipMembership(t *testing.T) { var eg errgroup.Group // Configure node1 - m1 := test.NewCommandNode(false) + m1 := test.NewCommandNode(t, false) defer m1.Close() eg.Go(func() error { m1.Config.Gossip.Port = "0" @@ -595,7 +595,7 @@ func TestCluster_GossipMembership(t *testing.T) { }) // Configure node1 - m2 := test.NewCommandNode(false) + m2 := test.NewCommandNode(t, false) defer m2.Close() eg.Go(func() error { m2.Config.Gossip.Port = "0" @@ -630,8 +630,8 @@ func TestCluster_GossipMembership(t *testing.T) { func TestClusterResize_RemoveNode(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() - m0 := cluster[0] - m1 := cluster[1] + m0 := cluster.GetNode(0) + m1 := cluster.GetNode(1) mustNodeID := func(baseURL string) string { body := test.Do(t, "GET", fmt.Sprintf("%s/status", baseURL), "").Body @@ -730,7 +730,7 @@ func TestClusterMutualTLS(t *testing.T) { cluster := test.MustRunCluster(t, 3, commandOpts...) defer cluster.Close() - m0 := cluster[0] + m0 := cluster.GetNode(0) client0 := m0.Client() if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { diff --git a/server/grpc.go b/server/grpc.go index af11b6f3c..69111ef82 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -21,6 +21,7 @@ import ( "io" "net" "strings" + "sync" "time" "github.com/pilosa/pilosa/v2" @@ -909,6 +910,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe type grpcServer struct { api *pilosa.API + mu sync.Mutex grpcServer *grpc.Server ln net.Listener @@ -956,11 +958,13 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error { } // create grpc server + s.mu.Lock() s.grpcServer = grpc.NewServer(opts...) pb.RegisterPilosaServer(s.grpcServer, NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats)) // register the server so its services are available to grpc_cli and others reflection.Register(s.grpcServer) + s.mu.Unlock() // and start... if err := s.grpcServer.Serve(s.ln); err != nil { @@ -969,6 +973,16 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error { return nil } +// Stop stops the GRPC server. There's no error because the underlying GRPC +// stuff doesn't report an error. +func (s *grpcServer) Stop() { + s.mu.Lock() + defer s.mu.Unlock() + if s.grpcServer != nil { + s.grpcServer.Stop() + } +} + func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { server := &grpcServer{ logger: logger.NopLogger, diff --git a/server/grpc_test.go b/server/grpc_test.go index 4529c0df8..77c030e23 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -709,6 +709,7 @@ func TestQuerySQLUnary(t *testing.T) { {"Table", "string"}, }, rows: []row{ + {[]columnResponse{"delete_me"}}, {[]columnResponse{"grouper"}}, {[]columnResponse{"joiner"}}, }, @@ -731,6 +732,27 @@ func TestQuerySQLUnary(t *testing.T) { }, eq: equal, }, + { + sql: "drop table delete_me", + exp: tableResponse{ + headers: []columnInfo{}, + rows: []row{}, + }, + eq: equal, + }, + { + sql: "show tables", + exp: tableResponse{ + headers: []columnInfo{ + {"Table", "string"}, + }, + rows: []row{ + {[]columnResponse{"grouper"}}, + {[]columnResponse{"joiner"}}, + }, + }, + eq: equal, + }, } for i, test := range tests { @@ -881,6 +903,9 @@ func setUpTestQuerySQLUnary(ctx context.Context, t *testing.T) (gh *server.GRPCH } } + // delete_me + m.MustCreateIndex(t, "delete_me", pilosa.IndexOptions{TrackExistence: true}) + return gh, func() { if err := m.API.DeleteIndex(ctx, joiner.Name()); err != nil { panic(err) diff --git a/server/handler_test.go b/server/handler_test.go index ccd91bdcc..987dac24d 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -43,7 +43,7 @@ import ( func TestHandler_PostSchemaCluster(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) h := cmd.Handler.(*http.Handler).Handler t.Run("PostSchema", func(t *testing.T) { @@ -56,8 +56,8 @@ func TestHandler_PostSchemaCluster(t *testing.T) { } t.Fatalf("unexpected code: %v, bod: %s", w.Code, bod) } - for i := 0; i < len(cluster); i++ { - cmd = cluster[i] + for i := 0; i < cluster.Len(); i++ { + cmd = cluster.GetNode(i) idx, err := cmd.API.Index(context.Background(), "blah") if err != nil { t.Fatalf("getting index: %v", err) @@ -82,7 +82,7 @@ func TestHandler_PostSchemaCluster(t *testing.T) { func TestHandler_Endpoints(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} @@ -1049,7 +1049,7 @@ func TestHandler_Endpoints(t *testing.T) { clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})}) defer clus.Close() w = httptest.NewRecorder() - h := clus[0].Handler.(*http.Handler).Handler + h := clus.GetNode(0).Handler.(*http.Handler).Handler h.ServeHTTP(w, req) result = w.Result() @@ -1297,59 +1297,59 @@ func TestHandler_Endpoints(t *testing.T) { } func TestCluster_TranslateStore(t *testing.T) { - cluster := make(test.Cluster, 1) - cluster[0] = test.NewCommandNode(true, + cluster := test.MustNewCluster(t, 1) + cluster.Nodes[0] = test.NewCommandNode(t, true, server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), ), ) - cluster[0].Config.Gossip.Port = "0" - err := cluster[0].Start() + cluster.GetNode(0).Config.Gossip.Port = "0" + err := cluster.GetNode(0).Start() if err != nil { t.Fatalf("starting cluster 0: %v", err) } - defer cluster[0].Close() + defer cluster.GetNode(0).Close() - test.Do(t, "POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}") + test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0", "{\"options\": {\"keys\": true}}") } func TestClusterTranslator(t *testing.T) { - cluster := make(test.Cluster, 2) - cluster[0] = test.NewCommandNode(true, + cluster := test.MustNewCluster(t, 2) + cluster.Nodes[0] = test.NewCommandNode(t, true, server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), ), ) - cluster[0].Config.Gossip.Port = "0" - err := cluster[0].Start() + cluster.GetNode(0).Config.Gossip.Port = "0" + err := cluster.GetNode(0).Start() if err != nil { t.Fatalf("starting cluster 0: %v", err) } - defer cluster[0].Close() - cluster[1] = test.NewCommandNode(false, + defer cluster.GetNode(0).Close() + cluster.Nodes[1] = test.NewCommandNode(t, false, server.OptCommandServerOptions( pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderWithLockerFunc(nil, &sync.Mutex{})), ), ) - cluster[1].Config.Gossip.Port = "0" - cluster[1].Config.Gossip.Seeds = []string{cluster[0].GossipAddress()} - err = cluster[1].Start() + cluster.GetNode(1).Config.Gossip.Port = "0" + cluster.GetNode(1).Config.Gossip.Seeds = []string{cluster.GetNode(0).GossipAddress()} + err = cluster.GetNode(1).Start() if err != nil { t.Fatalf("starting cluster 1: %v", err) } - defer cluster[1].Close() + defer cluster.GetNode(1).Close() - test.Do(t, "POST", cluster[0].URL()+"/index/i0", "{\"options\": {\"keys\": true}}") - test.Do(t, "POST", cluster[0].URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}") + test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0", "{\"options\": {\"keys\": true}}") + test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0/field/f0", "{\"options\": {\"keys\": true}}") - test.Do(t, "POST", cluster[0].URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")") + test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0/query", "Set(\"foo\", f0=\"bar\")") var result0, result1 string if err := test.RetryUntil(2*time.Second, func() error { - result0 = test.Do(t, "POST", cluster[0].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body - result1 = test.Do(t, "POST", cluster[1].URL()+"/index/i0/query", "Row(f0=\"bar\")").Body + result0 = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/i0/query", "Row(f0=\"bar\")").Body + result1 = test.Do(t, "POST", cluster.GetNode(1).URL()+"/index/i0/query", "Row(f0=\"bar\")").Body if result0 != result1 { return fmt.Errorf("`%s` != `%s`", result0, result1) } diff --git a/server/server.go b/server/server.go index e74d460e2..90061a1b4 100644 --- a/server/server.go +++ b/server/server.go @@ -49,6 +49,7 @@ import ( "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/statsd" "github.com/pilosa/pilosa/v2/syswrap" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" ) @@ -196,6 +197,7 @@ func (m *Command) Start() (err error) { } } + _ = testhook.Opened(pilosa.NewAuditor(), m, nil) close(m.Started) return nil } @@ -537,6 +539,7 @@ func (m *Command) GossipTransport() *gossip.Transport { func (m *Command) Close() error { defer close(m.done) eg := errgroup.Group{} + m.grpcServer.Stop() eg.Go(m.Handler.Close) eg.Go(m.Server.Close) eg.Go(m.API.Close) @@ -552,6 +555,7 @@ func (m *Command) Close() error { } err := eg.Wait() + _ = testhook.Closed(pilosa.NewAuditor(), m, nil) return errors.Wrap(err, "closing everything") } diff --git a/server/server_test.go b/server/server_test.go index 3534fe0d0..4b6b61eb2 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -355,7 +355,7 @@ func TestConcurrentFieldCreation(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() - api0 := cluster[0].API + api0 := cluster.GetNode(0).API if _, err := api0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil { t.Fatalf("creating index: %v", err) } @@ -379,10 +379,10 @@ func TestTransactionsAPI(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() - api0 := cluster[0].API - api1 := cluster[1].API + api0 := cluster.GetNode(0).API + api1 := cluster.GetNode(1).API ctx := context.Background() - //api2 := cluster[2].API + //api2 := cluster.GetNode(2).API // can fetch empty transactions if trnsMap, err := api0.Transactions(ctx); err != nil { @@ -508,7 +508,7 @@ func TestMain_RecalculateHashes(t *testing.T) { defer cluster.Close() // Create the schema. - client0 := cluster[0].Client() + client0 := cluster.GetNode(0).Client() if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { t.Fatal("create index:", err) } @@ -523,12 +523,12 @@ func TestMain_RecalculateHashes(t *testing.T) { data = append(data, fmt.Sprintf(`Set(%d, f=%d)`, columnID, rowID)) } } - if _, err := cluster[0].Query(t, "i", "", strings.Join(data, "")); err != nil { + if _, err := cluster.GetNode(0).Query(t, "i", "", strings.Join(data, "")); err != nil { t.Fatal("setting columns:", err) } // Calculate caches on the first node - err := cluster[0].RecalculateCaches(t) + err := cluster.GetNode(0).RecalculateCaches(t) if err != nil { t.Fatalf("recalculating caches: %v", err) } @@ -536,7 +536,7 @@ func TestMain_RecalculateHashes(t *testing.T) { target := `{"results":[[{"id":7,"key":"","count":99},{"id":1,"key":"","count":99},{"id":9,"key":"","count":99},{"id":5,"key":"","count":99},{"id":4,"key":"","count":99},{"id":8,"key":"","count":99},{"id":2,"key":"","count":99},{"id":6,"key":"","count":99},{"id":3,"key":"","count":99}]]}` // Run a TopN query on all nodes. The result should be the same as the target. - for _, m := range cluster { + for _, m := range cluster.Nodes { res, err := m.Query(t, "i", "", `TopN(f)`) if err != nil { t.Fatal(err) @@ -635,27 +635,27 @@ func TestClusteringNodesReplica1(t *testing.T) { t.Fatalf("starting cluster: %v", err) } - if err := cluster[2].Command.Close(); err != nil { + if err := cluster.GetNode(2).Command.Close(); err != nil { t.Fatalf("closing third node: %v", err) } // confirm that cluster stops accepting queries after one node closes - if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") { + if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") { t.Fatalf("got unexpected error querying an incomplete cluster: %v", err) } // Create new main with the same config. - config := cluster[2].Command.Config + config := cluster.GetNode(2).Command.Config config.Translation.MapSize = 100000 // this isn't necessary, but makes the test run way faster - config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port)) + config.Gossip.Port = strconv.Itoa(int(cluster.GetNode(2).Command.GossipTransport().URI.Port)) - cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore))) - cluster[2].Command.Config = config + cluster.GetNode(2).Command = server.NewCommand(cluster.GetNode(2).Stdin, cluster.GetNode(2).Stdout, cluster.GetNode(2).Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore))) + cluster.GetNode(2).Command.Config = config // Run new program. - if err := cluster[2].Start(); err != nil { + if err := cluster.GetNode(2).Start(); err != nil { t.Fatalf("restarting node 2: %v", err) } @@ -667,7 +667,7 @@ func TestClusteringNodesReplica1(t *testing.T) { func TestClusteringNodesReplica2(t *testing.T) { cluster := test.MustNewCluster(t, 3) - for _, c := range cluster { + for _, c := range cluster.Nodes { c.Config.Cluster.ReplicaN = 2 } err := cluster.Start() @@ -681,7 +681,7 @@ func TestClusteringNodesReplica2(t *testing.T) { t.Fatalf("starting cluster: %v", err) } - if err := cluster[2].Command.Close(); err != nil { + if err := cluster.GetNode(2).Command.Close(); err != nil { t.Fatalf("closing third node: %v", err) } @@ -691,12 +691,12 @@ func TestClusteringNodesReplica2(t *testing.T) { } // confirm that cluster keeps accepting queries if replication > 1 - if _, err := cluster[0].API.CreateIndex(context.Background(), "anewindex", pilosa.IndexOptions{}); err != nil { + if _, err := cluster.GetNode(0).API.CreateIndex(context.Background(), "anewindex", pilosa.IndexOptions{}); err != nil { t.Fatalf("got unexpected error creating index: %v", err) } // confirm that cluster stops accepting queries if 2 nodes fail and replication == 2 - if err := cluster[1].Command.Close(); err != nil { + if err := cluster.GetNode(1).Command.Close(); err != nil { t.Fatalf("closing 2nd node: %v", err) } @@ -705,23 +705,23 @@ func TestClusteringNodesReplica2(t *testing.T) { t.Fatalf("after closing second server: %v", err) } - if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") { + if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") { t.Fatalf("got unexpected error querying an incomplete cluster: %v", err) } // Create new main with the same config. - config := cluster[2].Command.Config + config := cluster.GetNode(2).Command.Config config.Translation.MapSize = 100000 - // config.Bind = cluster[2].API.Node().URI.HostPort() + // config.Bind = cluster.GetNode(2).API.Node().URI.HostPort() // this isn't necessary, but makes the test run way faster - config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port)) + config.Gossip.Port = strconv.Itoa(int(cluster.GetNode(2).Command.GossipTransport().URI.Port)) - cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore))) - cluster[2].Command.Config = config + cluster.GetNode(2).Command = server.NewCommand(cluster.GetNode(2).Stdin, cluster.GetNode(2).Stdout, cluster.GetNode(2).Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore))) + cluster.GetNode(2).Command.Config = config // Run new program. - if err := cluster[2].Start(); err != nil { + if err := cluster.GetNode(2).Start(); err != nil { t.Fatalf("restarting node 2: %v", err) } @@ -731,18 +731,18 @@ func TestClusteringNodesReplica2(t *testing.T) { } // Create new main with the same config. - config = cluster[1].Command.Config - // config.Bind = cluster[1].API.Node().URI.HostPort() + config = cluster.GetNode(1).Command.Config + // config.Bind = cluster.GetNode(1).API.Node().URI.HostPort() config.Translation.MapSize = 100000 // this isn't necessary, but makes the test run way faster - config.Gossip.Port = strconv.Itoa(int(cluster[1].Command.GossipTransport().URI.Port)) + config.Gossip.Port = strconv.Itoa(int(cluster.GetNode(1).Command.GossipTransport().URI.Port)) - cluster[1].Command = server.NewCommand(cluster[1].Stdin, cluster[1].Stdout, cluster[1].Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore))) - cluster[1].Command.Config = config + cluster.GetNode(1).Command = server.NewCommand(cluster.GetNode(1).Stdin, cluster.GetNode(1).Stdout, cluster.GetNode(1).Stderr, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore))) + cluster.GetNode(1).Command.Config = config // Run new program. - if err := cluster[1].Start(); err != nil { + if err := cluster.GetNode(1).Start(); err != nil { t.Fatalf("restarting node 1: %v", err) } @@ -754,7 +754,7 @@ func TestClusteringNodesReplica2(t *testing.T) { func TestRemoveNodeAfterItDies(t *testing.T) { cluster := test.MustNewCluster(t, 3) - for _, c := range cluster { + for _, c := range cluster.Nodes { c.Config.Cluster.ReplicaN = 2 } err := cluster.Start() @@ -774,10 +774,9 @@ func TestRemoveNodeAfterItDies(t *testing.T) { t.Fatalf("starting cluster: %v", err) } - // prevent double-closing cluster[2] from the deferred Close above - disabled, cluster := cluster[2], cluster[:2] - - if err := disabled.Command.Close(); err != nil { + // prevent double-closing cluster.GetNode(2) from the deferred Close above + disabled := cluster.GetNode(2) + if err := cluster.CloseAndRemove(2); err != nil { t.Fatalf("closing third node: %v", err) } @@ -786,7 +785,7 @@ func TestRemoveNodeAfterItDies(t *testing.T) { t.Fatalf("starting cluster: %v", err) } - if _, err := cluster[0].API.RemoveNode(disabled.API.Node().ID); err != nil { + if _, err := cluster.GetNode(0).API.RemoveNode(disabled.API.Node().ID); err != nil { t.Fatalf("removing failed node: %v", err) } @@ -795,7 +794,7 @@ func TestRemoveNodeAfterItDies(t *testing.T) { t.Fatalf("removing disabled node: %v", err) } - hosts := cluster[0].API.Hosts(context.Background()) + hosts := cluster.GetNode(0).API.Hosts(context.Background()) if len(hosts) != 2 { t.Fatalf("unexpected hosts: %v", hosts) } @@ -803,7 +802,7 @@ func TestRemoveNodeAfterItDies(t *testing.T) { func TestRemoveConcurrentIndexCreation(t *testing.T) { cluster := test.MustNewCluster(t, 3) - for _, c := range cluster { + for _, c := range cluster.Nodes { c.Config.Cluster.ReplicaN = 2 } err := cluster.Start() @@ -818,11 +817,11 @@ func TestRemoveConcurrentIndexCreation(t *testing.T) { errc := make(chan error) go func() { - _, err := cluster[0].API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{}) + _, err := cluster.GetNode(0).API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{}) errc <- err }() - if _, err := cluster[0].API.RemoveNode(cluster[2].API.Node().ID); err != nil { + if _, err := cluster.GetNode(0).API.RemoveNode(cluster.GetNode(2).API.Node().ID); err != nil { t.Fatalf("removing node: %v", err) } @@ -831,7 +830,7 @@ func TestRemoveConcurrentIndexCreation(t *testing.T) { t.Fatalf("starting cluster: %v", err) } - hosts := cluster[0].API.Hosts(context.Background()) + hosts := cluster.GetNode(0).API.Hosts(context.Background()) if len(hosts) != 2 { t.Fatalf("unexpected hosts: %v", hosts) } @@ -948,9 +947,9 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) { func TestClusterQueriesAfterRestart(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() - cmd1 := cluster[1] + cmd1 := cluster.GetNode(1) - for _, com := range cluster { + for _, com := range cluster.Nodes { nodes := com.API.Hosts(context.Background()) for _, n := range nodes { if n.State != "READY" { @@ -992,7 +991,7 @@ func TestClusterQueriesAfterRestart(t *testing.T) { } // confirm that cluster stops accepting queries after one node closes - if _, err := cluster[0].API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") { + if _, err := cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state STARTING") { t.Fatalf("got unexpected error querying an incomplete cluster: %v", err) } @@ -1034,9 +1033,9 @@ func TestClusterExhaustingConnections(t *testing.T) { } cluster := test.MustRunCluster(t, 5) defer cluster.Close() - cmd1 := cluster[1] + cmd1 := cluster.GetNode(1) - for _, com := range cluster { + for _, com := range cluster.Nodes { nodes := com.API.Hosts(context.Background()) for _, n := range nodes { if n.State != "READY" { @@ -1053,7 +1052,7 @@ func TestClusterExhaustingConnections(t *testing.T) { i := i eg.Go(func() error { for j := i; j < 10000; j += 20 { - _, err := cluster[i%5].API.Query(context.Background(), &pilosa.QueryRequest{ + _, err := cluster.GetNode(i%5).API.Query(context.Background(), &pilosa.QueryRequest{ Index: "testidx", Query: fmt.Sprintf("Set(%d, testfield=0)", j*pilosa.ShardWidth), }) @@ -1120,9 +1119,9 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) { } cluster := test.MustRunCluster(t, 5) defer cluster.Close() - cmd1 := cluster[1] + cmd1 := cluster.GetNode(1) - for _, com := range cluster { + for _, com := range cluster.Nodes { nodes := com.API.Hosts(context.Background()) for _, n := range nodes { if n.State != "READY" { @@ -1151,7 +1150,7 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) { if (j-i)%1000 == 0 { fmt.Printf("%d is %.2f%% done.\n", i, float64(j-i)*100/100000) } - err := cluster[i%5].API.ImportRoaring(context.Background(), "testidx", "testfield", j, false, &pilosa.ImportRoaringRequest{ + err := cluster.GetNode(int(i%5)).API.ImportRoaring(context.Background(), "testidx", "testfield", j, false, &pilosa.ImportRoaringRequest{ Views: map[string][]byte{ "": data, }, @@ -1172,12 +1171,12 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) { func TestClusterMinMaxSumDecimal(t *testing.T) { cluster := test.MustRunCluster(t, 3) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) cmd.MustCreateIndex(t, "testdec", pilosa.IndexOptions{Keys: true, TrackExistence: true}) cmd.MustCreateField(t, "testdec", "adec", pilosa.OptFieldTypeDecimal(2)) - test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", ` + test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", ` Set("a", adec=42.2) Set("b", adec=11.12) Set("c", adec=13.41) @@ -1188,21 +1187,21 @@ Set("g", adec=15.52) Set("h", adec=100.22) `) - result := test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", "Sum(field=adec)") + result := test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Sum(field=adec)") if !strings.Contains(result.Body, `"decimalValue":305.59`) { t.Fatalf("expected decimal sum of 305.59, but got: '%s'", result.Body) } else if !strings.Contains(result.Body, `"count":8`) { t.Fatalf("expected count 8, but got: '%s'", result.Body) } - result = test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", "Max(field=adec)") + result = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Max(field=adec)") if !strings.Contains(result.Body, `"decimalValue":100.22`) { t.Fatalf("expected decimal max of 100.22, but got: '%s'", result.Body) } else if !strings.Contains(result.Body, `"count":1`) { t.Fatalf("expected count 1, but got: '%s'", result.Body) } - result = test.Do(t, "POST", cluster[0].URL()+"/index/testdec/query", "Min(field=adec)") + result = test.Do(t, "POST", cluster.GetNode(0).URL()+"/index/testdec/query", "Min(field=adec)") if !strings.Contains(result.Body, `"decimalValue":11.12`) { t.Fatalf("expected decimal min of 11.12, but got: '%s'", result.Body) } else if !strings.Contains(result.Body, `"count":1`) { diff --git a/server/sql.go b/server/sql.go index 0f1de916c..7647e0377 100644 --- a/server/sql.go +++ b/server/sql.go @@ -38,17 +38,14 @@ func execSQL(ctx context.Context, api *pilosa.API, logger logger.Logger, querySt case sql.SQLTypeSelect: handler := sql.NewSelectHandler(api) results, err = handler.Handle(ctx, query) - if err != nil { - return nil, errors.Wrap(err, "failed to start SQL query") - } case sql.SQLTypeShow: handler := sql.NewShowHandler(api) results, err = handler.Handle(ctx, query) - if err != nil { - return nil, errors.Wrap(err, "failed to start SQL query") - } + case sql.SQLTypeEmpty: + handler := sql.NewDDLHandler(api) + results, err = handler.Handle(ctx, query) default: return nil, status.Errorf(codes.Unimplemented, "query type not supported") } - return results, nil + return results, errors.Wrap(err, "failed to start SQL query") } diff --git a/server_internal_test.go b/server_internal_test.go index 3a302685a..f799629b5 100644 --- a/server_internal_test.go +++ b/server_internal_test.go @@ -15,10 +15,11 @@ package pilosa import ( - "io/ioutil" "runtime" "testing" "time" + + "github.com/pilosa/pilosa/v2/testhook" ) // Ensure the file handle count is working @@ -40,7 +41,7 @@ func TestCountOpenFiles(t *testing.T) { func TestMonitorAntiEntropyZero(t *testing.T) { - td, err := ioutil.TempDir(*TempDir, "") + td, err := testhook.TempDirInDir(t, *TempDir, "") if err != nil { t.Fatalf("getting temp dir: %v", err) } @@ -49,6 +50,7 @@ func TestMonitorAntiEntropyZero(t *testing.T) { if err != nil { t.Fatalf("making new server: %v", err) } + defer s.Close() ch := make(chan struct{}) go func() { diff --git a/snapshotqueue.go b/snapshotqueue.go index 1bdcdb7a9..4fc3e4f79 100644 --- a/snapshotqueue.go +++ b/snapshotqueue.go @@ -25,6 +25,7 @@ import ( "time" "github.com/pilosa/pilosa/v2/logger" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" ) @@ -90,7 +91,7 @@ var defaultSnapshotQueue = &queuelessSnapshotQueue{} // w worker threads. func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue { ctx, cancel := context.WithCancel(context.Background()) - sq := prioritySnapshotQueue{ + sq := &prioritySnapshotQueue{ normal: make(chan snapshotRequest, n), urgent: make(chan snapshotRequest), background: make(chan snapshotRequest), @@ -102,8 +103,9 @@ func newSnapshotQueue(n int, w int, l logger.Logger) SnapshotQueue { if sq.logger == nil { sq.logger = logger.NewStandardLogger(os.Stderr) } + _ = testhook.Opened(NewAuditor(), sq, nil) sq.spawnWorkers(w) - return &sq + return sq } type snapshotRequest struct { @@ -136,6 +138,7 @@ type prioritySnapshotQueue struct { enqueued uint32 skipped uint32 } + stopped bool } func (sq *prioritySnapshotQueue) spawnWorkers(w int) { @@ -205,6 +208,10 @@ func (sq *prioritySnapshotQueue) process(req snapshotRequest) { func (sq *prioritySnapshotQueue) Stop() { sq.mu.Lock() defer sq.mu.Unlock() + if sq.stopped { + return + } + sq.stopped = true sq.cancel() // scanners need to be done before we close the other channels. sq.scanWG.Wait() @@ -214,6 +221,7 @@ func (sq *prioritySnapshotQueue) Stop() { sq.urgent = nil close(sq.background) sq.background = nil + _ = testhook.Closed(NewAuditor(), sq, nil) enqueued := atomic.LoadUint32(&sq.stats.enqueued) skipped := atomic.LoadUint32(&sq.stats.skipped) if skipped > 0 || enqueued > 1 { diff --git a/sql/ddl.go b/sql/ddl.go new file mode 100644 index 000000000..3b693850e --- /dev/null +++ b/sql/ddl.go @@ -0,0 +1,65 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sql + +import ( + "context" + "fmt" + + "github.com/pilosa/pilosa/v2" + pproto "github.com/pilosa/pilosa/v2/proto" + "github.com/pkg/errors" + "vitess.io/vitess/go/vt/sqlparser" +) + +// DDLHandler executes CREATE, ALTER, DROP, RENAME, TRUNCATE or ANALYZE statement. +type DDLHandler struct { + api *pilosa.API +} + +// NewDDLHandler constructor +func NewDDLHandler(api *pilosa.API) *DDLHandler { + return &DDLHandler{ + api: api, + } +} + +// Handle executes mapped SQL +func (h *DDLHandler) Handle(ctx context.Context, mapped *MappedSQL) (pproto.StreamClient, error) { + stmt, ok := mapped.Statement.(*sqlparser.DDL) + if !ok { + return nil, fmt.Errorf("statement is not type DDL: %T", mapped.Statement) + } + + switch stmt.Action { + case sqlparser.DropStr: + return h.execDropTable(ctx, stmt) + + default: + return nil, errors.Errorf("unsupported DDL action: %s", stmt.Action) + } +} + +func (h *DDLHandler) execDropTable(ctx context.Context, stmt *sqlparser.DDL) (pproto.StreamClient, error) { + if n := len(stmt.FromTables); n != 1 { + return nil, fmt.Errorf("statement can only contain a single drop table, but got: %d", n) + } + + indexName := stmt.FromTables[0].ToViewName().Name.String() + if err := h.api.DeleteIndex(ctx, indexName); err != nil { + return nil, errors.Wrapf(err, "deleting index %s", indexName) + } + return pproto.EmptyStream{}, nil +} diff --git a/stats/stats_test.go b/stats/stats_test.go index 8e9cb8de2..842a72f93 100644 --- a/stats/stats_test.go +++ b/stats/stats_test.go @@ -32,7 +32,7 @@ import ( // TestMultiStatClient_Expvar run the multistat client with exp var // since the EXPVAR data is stored in a global we should run these in one test function func TestMultiStatClient_Expvar(t *testing.T) { - hldr := test.MustOpenHolder() + hldr := test.MustOpenHolder(t) defer hldr.Close() c := stats.NewExpvarStatsClient() @@ -93,7 +93,7 @@ func TestMultiStatClient_Expvar(t *testing.T) { func TestStatsCount_TopN(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()} hldr.SetBit("d", "f", 0, 0) hldr.SetBit("d", "f", 0, 1) @@ -115,7 +115,7 @@ func TestStatsCount_TopN(t *testing.T) { called = true }, } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil { t.Fatal(err) } if !called { @@ -126,7 +126,7 @@ func TestStatsCount_TopN(t *testing.T) { func TestStatsCount_Bitmap(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()} hldr.SetBit("d", "f", 0, 0) hldr.SetBit("d", "f", 0, 1) @@ -144,7 +144,7 @@ func TestStatsCount_Bitmap(t *testing.T) { called = true }, } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil { t.Fatal(err) } if !called { @@ -155,7 +155,7 @@ func TestStatsCount_Bitmap(t *testing.T) { func TestStatsCount_SetRowAttrsBulk(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()} hldr.SetBit("d", "f", 10, 0) hldr.SetBit("d", "f", 10, 1) @@ -178,7 +178,7 @@ func TestStatsCount_SetRowAttrsBulk(t *testing.T) { called = true }, } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetRowAttrs(f, 10, foo="bar")`}); err != nil { t.Fatal(err) } if !called { @@ -189,7 +189,7 @@ func TestStatsCount_SetRowAttrsBulk(t *testing.T) { func TestStatsCount_SetColumnAttrs(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()} hldr.SetBit("d", "f", 10, 0) hldr.SetBit("d", "f", 10, 1) @@ -212,7 +212,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { called = true }, } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetColumnAttrs(10, foo="bar")`}); err != nil { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `SetColumnAttrs(10, foo="bar")`}); err != nil { t.Fatal(err) } if !called { @@ -223,7 +223,7 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { func TestStatsCount_APICalls(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() - cmd := cluster[0] + cmd := cluster.GetNode(0) h := cmd.Handler.(*http.Handler).Handler holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} diff --git a/test/cluster.go b/test/cluster.go index 4fbc2050e..e0c2658aa 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -35,20 +35,34 @@ type ModHasher struct{} func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n } // Cluster represents a Pilosa cluster (multiple Command instances) -type Cluster []*Command +type Cluster struct { + Nodes []*Command +} // Query executes an API.Query through one of the cluster's node's API. It fails // the test if there is an error. -func (c Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse { +func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse { t.Helper() - if len(c) == 0 { + if len(c.Nodes) == 0 { t.Fatal("must have at least one node in cluster to query") } - return c[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query}) + return c.Nodes[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query}) } -func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) { +func (c *Cluster) GetNode(n int) *Command { + return c.Nodes[n] +} + +func (c *Cluster) GetHolder(n int) *Holder { + return &Holder{Holder: c.Nodes[n].Server.Holder()} +} + +func (c *Cluster) Len() int { + return len(c.Nodes) +} + +func (c *Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) { t.Helper() byShard := make(map[uint64][][2]uint64) for _, rowcol := range rowcols { @@ -63,7 +77,7 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint rowIDs[i] = bit[0] colIDs[i] = bit[1] } - nodes, err := c[0].API.ShardNodes(context.Background(), index, shard) + nodes, err := c.Nodes[0].API.ShardNodes(context.Background(), index, shard) if err != nil { t.Fatalf("getting shard nodes: %v", err) } @@ -72,7 +86,7 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint // suggesting that elsewhere we would support importing to a // single node, regardless of where the data ends up. for _, node := range nodes { - for _, com := range c { + for _, com := range c.Nodes { if com.API.Node().ID != node.ID { continue } @@ -92,13 +106,13 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint } // CreateField creates the index (if necessary) and field specified. -func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field { +func (c *Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field { t.Helper() - idx, err := c[0].API.CreateIndex(context.Background(), index, iopts) + idx, err := c.Nodes[0].API.CreateIndex(context.Background(), index, iopts) if err != nil && !strings.Contains(err.Error(), "index already exists") { t.Fatalf("creating index: %v", err) } else if err != nil { // index exists - idx, err = c[0].API.Index(context.Background(), index) + idx, err = c.Nodes[0].API.Index(context.Background(), index) if err != nil { t.Fatalf("getting index: %v", err) } @@ -107,7 +121,7 @@ func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptio t.Logf("existing index options:\n%v\ndon't match given opts:\n%v\n in pilosa/test.Cluster.CreateField", idx.Options(), iopts) } - f, err := c[0].API.CreateField(context.Background(), index, field, fopts...) + f, err := c.Nodes[0].API.CreateField(context.Background(), index, field, fopts...) // we'll assume the field doesn't exist because checking if the options // match seems painful. if err != nil { @@ -117,9 +131,9 @@ func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptio } // Start runs a Cluster -func (c Cluster) Start() error { - var gossipSeeds = make([]string, len(c)) - for i, cc := range c { +func (c *Cluster) Start() error { + var gossipSeeds = make([]string, len(c.Nodes)) + for i, cc := range c.Nodes { cc.Config.Gossip.Port = "0" cc.Config.Gossip.Seeds = gossipSeeds[:i] if err := cc.Start(); err != nil { @@ -131,8 +145,8 @@ func (c Cluster) Start() error { } // Stop stops a Cluster -func (c Cluster) Close() error { - for i, cc := range c { +func (c *Cluster) Close() error { + for i, cc := range c.Nodes { if err := cc.Close(); err != nil { return errors.Wrapf(err, "stopping server %d", i) } @@ -140,19 +154,30 @@ func (c Cluster) Close() error { return nil } +func (c *Cluster) CloseAndRemove(n int) error { + if n < 0 || n >= len(c.Nodes) { + return fmt.Errorf("close/remove from cluster: index %d out of range (len %d)", n, len(c.Nodes)) + } + err := c.Nodes[n].Close() + copy(c.Nodes[n:], c.Nodes[n+1:]) + c.Nodes = c.Nodes[:len(c.Nodes)-1] + return err +} + // AwaitState waits for the cluster coordinator (assumed to be the first // node) to reach a specified state. -func (c Cluster) AwaitCoordinatorState(expectedState string, timeout time.Duration) error { - if len(c) < 1 { +func (c *Cluster) AwaitCoordinatorState(expectedState string, timeout time.Duration) error { + if len(c.Nodes) < 1 { return errors.New("can't await coordinator state on an empty cluster") } - return c[:1].AwaitState(expectedState, timeout) + onlyCoordinator := &Cluster{Nodes: c.Nodes[:1]} + return onlyCoordinator.AwaitState(expectedState, timeout) } // ExceptionalState returns an error if any node in the cluster is not // in the expected state. -func (c Cluster) ExceptionalState(expectedState string) error { - for _, node := range c { +func (c *Cluster) ExceptionalState(expectedState string) error { + for _, node := range c.Nodes { state := node.API.State() if state != expectedState { return fmt.Errorf("node %q: state %s", node.ID(), state) @@ -162,8 +187,8 @@ func (c Cluster) ExceptionalState(expectedState string) error { } // AwaitState waits for the whole cluster to reach a specified state. -func (c Cluster) AwaitState(expectedState string, timeout time.Duration) (err error) { - if len(c) < 1 { +func (c *Cluster) AwaitState(expectedState string, timeout time.Duration) (err error) { + if len(c.Nodes) < 1 { return errors.New("can't await state of an empty cluster") } startTime := time.Now() @@ -184,7 +209,7 @@ func (c Cluster) AwaitState(expectedState string, timeout time.Duration) (err er // slice of command options, those options are used with every node. // If it is empty, default options are used. Otherwise, it must contain size // slices of command options, which are used with corresponding nodes. -func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster { +func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cluster { tb.Helper() c, err := newCluster(tb, size, opts...) if err != nil { @@ -206,7 +231,7 @@ func CheckClusterState(m *Command, state string, n int) bool { } // newCluster creates a new cluster -func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluster, error) { +func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Cluster, error) { if size == 0 { return nil, errors.New("cluster must contain at least one node") } @@ -214,26 +239,26 @@ func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluste return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes") } - cluster := make(Cluster, size) + cluster := &Cluster{Nodes: make([]*Command, size)} name := tb.Name() for i := 0; i < size; i++ { var commandOpts []server.CommandOption if len(opts) > 0 { commandOpts = opts[i%len(opts)] } - m := NewCommandNode(i == 0, commandOpts...) + m := NewCommandNode(tb, i == 0, commandOpts...) err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name+"_"+strconv.Itoa(i)), 0600) if err != nil { return nil, errors.Wrap(err, "writing node id") } - cluster[i] = m + cluster.Nodes[i] = m } return cluster, nil } // runCluster creates and starts a new cluster -func runCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluster, error) { +func runCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Cluster, error) { cluster, err := newCluster(tb, size, opts...) if err != nil { return nil, errors.Wrap(err, "new cluster") @@ -247,7 +272,7 @@ func runCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluste // MustRunCluster creates and starts a new cluster. The opts parameter // is slightly magical; see MustNewCluster. -func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster { +func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) *Cluster { // We want tests to default to using the in-memory translate store, so we // prepend opts with that functional option. If a different translate store // has been specified, it will override this one. diff --git a/test/field.go b/test/field.go index f56672834..3aacc02f6 100644 --- a/test/field.go +++ b/test/field.go @@ -15,11 +15,11 @@ package test import ( - "io/ioutil" "os" "testing" "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/testhook" ) // Field represents a test wrapper for pilosa.Field. @@ -28,8 +28,8 @@ type Field struct { } // newField returns a new instance of Field d/0. -func newField(opts pilosa.FieldOption) *Field { - path, err := ioutil.TempDir("", "pilosa-field-") +func newField(tb testing.TB, opts pilosa.FieldOption) *Field { + path, err := testhook.TempDir(tb, "pilosa-field-") if err != nil { panic(err) } @@ -41,8 +41,8 @@ func newField(opts pilosa.FieldOption) *Field { } // mustOpenField returns a new, opened field at a temporary path. Panic on error. -func mustOpenField(opts pilosa.FieldOption) *Field { - f := newField(opts) +func mustOpenField(tb testing.TB, opts pilosa.FieldOption) *Field { + f := newField(tb, opts) if err := f.Open(); err != nil { panic(err) } @@ -76,7 +76,7 @@ func (f *Field) reopen() error { // Ensure field can set its cache func TestField_SetCacheSize(t *testing.T) { - f := mustOpenField(pilosa.OptFieldTypeDefault()) + f := mustOpenField(t, pilosa.OptFieldTypeDefault()) defer f.close() cacheSize := uint32(100) diff --git a/test/holder.go b/test/holder.go index 8db4af11c..b45bcb6c3 100644 --- a/test/holder.go +++ b/test/holder.go @@ -15,14 +15,14 @@ package test import ( - "io/ioutil" "math" - "os" + "testing" "time" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/testhook" ) var panicOn = pilosa.PanicOn @@ -33,8 +33,8 @@ type Holder struct { } // NewHolder returns a new instance of Holder with a temporary path. -func NewHolder() *Holder { - path, err := ioutil.TempDir("", "pilosa-") +func NewHolder(tb testing.TB) *Holder { + path, err := testhook.TempDir(tb, "pilosa-holder-") if err != nil { panic(err) } @@ -47,17 +47,16 @@ func NewHolder() *Holder { } // MustOpenHolder creates and opens a holder at a temporary path. Panic on error. -func MustOpenHolder() *Holder { - h := NewHolder() +func MustOpenHolder(tb testing.TB) *Holder { + h := NewHolder(tb) if err := h.Open(); err != nil { panic(err) } return h } -// Close closes the holder and removes all underlying data. +// Close closes the holder. The data should be removed by the func (h *Holder) Close() error { - defer os.RemoveAll(h.Path) return h.Holder.Close() } diff --git a/test/index.go b/test/index.go index e51850d87..e65349f1a 100644 --- a/test/index.go +++ b/test/index.go @@ -15,10 +15,10 @@ package test import ( - "io/ioutil" - "os" + "testing" "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/testhook" ) // Index represents a test wrapper for pilosa.Index. @@ -27,12 +27,15 @@ type Index struct { } // newIndex returns a new instance of Index. -func newIndex() *Index { - path, err := ioutil.TempDir("", "pilosa-index-") +func newIndex(tb testing.TB) *Index { + path, err := testhook.TempDir(tb, "pilosa-index-") if err != nil { panic(err) } h := pilosa.NewHolder(pilosa.DefaultPartitionN) + testhook.Cleanup(tb, func() { + h.Close() + }) h.Path = path index, err := h.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { @@ -42,17 +45,13 @@ func newIndex() *Index { } // MustOpenIndex returns a new, opened index at a temporary path. Panic on error. -func MustOpenIndex() *Index { - index := newIndex() - if err := index.Open(false); err != nil { - panic(err) - } +func MustOpenIndex(tb testing.TB) *Index { + index := newIndex(tb) return index } // Close closes the index and removes the underlying data. func (i *Index) Close() error { - defer os.RemoveAll(i.Path()) return i.Index.Close() } @@ -70,10 +69,6 @@ func (i *Index) Reopen() error { if err != nil { return err } - - if err := i.Open(false); err != nil { - return err - } return nil } diff --git a/test/pilosa.go b/test/pilosa.go index e592f058e..f34cf0381 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -30,6 +30,7 @@ import ( "github.com/pilosa/pilosa/v2/encoding/proto" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/server" + "github.com/pilosa/pilosa/v2/testhook" ) //////////////////////////////////////////////////////////////////////////////////// @@ -48,8 +49,8 @@ func OptAllowedOrigins(origins []string) server.CommandOption { } // newCommand returns a new instance of Main with a temporary data directory and random port. -func newCommand(opts ...server.CommandOption) *Command { - path, err := ioutil.TempDir("", "pilosa-") +func newCommand(tb testing.TB, opts ...server.CommandOption) *Command { + path, err := testhook.TempDir(tb, "pilosa-command-") if err != nil { panic(err) } @@ -85,12 +86,12 @@ func newCommand(opts ...server.CommandOption) *Command { } // NewCommandNode returns a new instance of Command with clustering enabled. -func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command { +func NewCommandNode(tb testing.TB, isCoordinator bool, opts ...server.CommandOption) *Command { // We want tests to default to using the in-memory translate store, so we // prepend opts with that functional option. If a different translate store // has been specified, it will override this one. opts = prependTestServerOpts(opts) - m := newCommand(opts...) + m := newCommand(tb, opts...) m.Config.Cluster.Disabled = false m.Config.Cluster.Coordinator = isCoordinator return m @@ -99,7 +100,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command { // RunCommand returns a new, running Main. Panic on error. func RunCommand(t *testing.T) *Command { t.Helper() - m := newCommand(server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore))) + m := newCommand(t, server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore))) m.Config.Metric.Diagnostics = false // Disable diagnostics. m.Config.Gossip.Port = "0" if err := m.Start(); err != nil { @@ -307,8 +308,15 @@ func Do(t *testing.T, method, urlStr string, body string) *httpResponse { req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") - resp, err := gohttp.DefaultClient.Do(req) + // set a timeout instead of allowing gohttp.Defaultclient to + // potentially hang forever. + hc := &gohttp.Client{ + Timeout: time.Second * 10, + } + resp, err := hc.Do(req) + if err != nil { + fmt.Printf(" hc.Do() err = '%v'\n", err) t.Fatal(err) } defer resp.Body.Close() diff --git a/test/pilosa_test.go b/test/pilosa_test.go index 3ba9d432a..686c071e2 100644 --- a/test/pilosa_test.go +++ b/test/pilosa_test.go @@ -30,15 +30,15 @@ func TestNewCluster(t *testing.T) { cluster := test.MustRunCluster(t, numNodes) defer cluster.Close() - coordinator := getCoordinator(cluster[0]) + coordinator := getCoordinator(cluster.Nodes[0]) for i := 1; i < numNodes; i++ { - if coordi := getCoordinator(cluster[i]); coordi != coordinator { + if coordi := getCoordinator(cluster.Nodes[i]); coordi != coordinator { t.Fatalf("node %d does not have the same coordinator as node 0. '%v' and '%v' respectively", i, coordi, coordinator) } } req, err := http.NewRequest( "GET", - cluster[0].URL()+"/status", + cluster.Nodes[0].URL()+"/status", strings.NewReader(""), ) if err != nil { diff --git a/testhook/auditor.go b/testhook/auditor.go new file mode 100644 index 000000000..b7166664f --- /dev/null +++ b/testhook/auditor.go @@ -0,0 +1,168 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testhook + +import ( + "fmt" + "reflect" + "sync" +) + +// Auditor represents a thing which knows how to audit events. For instance, +// it can check on when things are accessed, or when they were opened, or +// whether opened objects are later closed. +type Auditor interface { + + // Registry yields a registry for objects of this type. + // multiple calls with the same object type yield the same registry. + Registry(interface{}) (Registry, error) + // Check performs any error-checking that can be done during + // usage. + Check() (error, []error) + // FinalCheck performs any error-checking that makes sense only + // after all operations are supposed to be complete, such as + // verifying that opened objects have been closed. + FinalCheck() (error, []error) +} + +// Created(a, o, kv) is shorthand for a.Registry(o).Created(o, kv) plus +// the error checking inside that. +func Created(a Auditor, o interface{}, kv KV) error { + r, err := a.Registry(o) + if err != nil { + return err + } + return r.Created(o, kv) +} + +// Opened(a, o, kv) is shorthand for a.Registry(o).Opened(o, kv) plus +// the error checking inside that. +func Opened(a Auditor, o interface{}, kv KV) error { + r, err := a.Registry(o) + if err != nil { + return err + } + return r.Opened(o, kv) +} + +// Closed(a, o, kv) is shorthand for a.Registry(o).Closed(o, kv) plus +// the error checking inside that. +func Closed(a Auditor, o interface{}, kv KV) error { + r, err := a.Registry(o) + if err != nil { + return err + } + return r.Closed(o, kv) +} + +// Destroyed(a, o, kv) is shorthand for a.Registry(o).Destroyed(o, kv) plus +// the error checking inside that. +func Destroyed(a Auditor, o interface{}, kv KV) error { + r, err := a.Registry(o) + if err != nil { + return err + } + return r.Destroyed(o, kv) +} + +// Seen(a, o, kv) is shorthand for a.Registry(o).Seen(o, kv) plus +// the error checking inside that. +func Seen(a Auditor, o interface{}, kv KV) error { + r, err := a.Registry(o) + if err != nil { + return err + } + return r.Seen(o, kv) +} + +// NopAuditor doesn't do anything. +type NopAuditor struct{} + +func (*NopAuditor) Registry(interface{}) (Registry, error) { + return NewNopRegistry(), nil +} + +func (*NopAuditor) Check() (error, []error) { + return nil, nil +} + +func (*NopAuditor) FinalCheck() (error, []error) { + return nil, nil +} + +func NewNopAuditor() *NopAuditor { + return &NopAuditor{} +} + +// VerifyCloseAuditor provides registries which it will check for things +// being closed. +type VerifyCloseAuditor struct { + registries map[reflect.Type]Registry + hooks RegistryHooks + regMu sync.Mutex +} + +func (v *VerifyCloseAuditor) Registry(o interface{}) (Registry, error) { + t := reflect.TypeOf(o) + v.regMu.Lock() + defer v.regMu.Unlock() + if exists, ok := v.registries[t]; ok { + return exists, nil + } + reg := NewSimpleRegistry(v.hooks[t]) + v.registries[t] = reg + return reg, nil +} + +func (*VerifyCloseAuditor) Check() (error, []error) { + return nil, nil +} + +func (v *VerifyCloseAuditor) FinalCheck() (error, []error) { + v.regMu.Lock() + defer v.regMu.Unlock() + var errs []error + for t, reg := range v.registries { + typeName := t.String() + live, err := reg.Live() + if err != nil { + errs = append(errs, fmt.Errorf("registry[%s]: retrieving live list: %v", + typeName, err)) + continue + } + if len(live) > 0 { + for addr, entry := range live { + if entry.Error != nil { + errs = append(errs, fmt.Errorf("%v: item created at %v, stack %s", + entry.Error, entry.Stamp, entry.Stack)) + } else { + errs = append(errs, fmt.Errorf("live item found at %p, created at %v, stack %s", + addr, entry.Stamp, entry.Stack)) + } + if entry.Data["stack"] != nil { + errs = append(errs, fmt.Errorf("stashed stack: %s", entry.Data["stack"])) + } + } + } + } + if len(errs) > 0 { + return fmt.Errorf("final check: %d error(s)", len(errs)), errs + } + return nil, nil +} + +func NewVerifyCloseAuditor(hooks RegistryHooks) *VerifyCloseAuditor { + return &VerifyCloseAuditor{registries: map[reflect.Type]Registry{}, hooks: hooks} +} diff --git a/testhook/auditor_test.go b/testhook/auditor_test.go new file mode 100644 index 000000000..deff64f88 --- /dev/null +++ b/testhook/auditor_test.go @@ -0,0 +1,331 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testhook_test + +import ( + "errors" + "reflect" + "testing" + + "github.com/pilosa/pilosa/v2/testhook" +) + +func TestAuditor_CatchError(t *testing.T) { + auditor := testhook.NewVerifyCloseAuditor(nil) + var x, y int + reg, err := auditor.Registry(&x) + if err != nil { + t.Fatalf("requesting registry: %v", err) + } + err = reg.Created(&x, nil) + if err != nil { + t.Fatalf("creating x: %v", err) + } + err = reg.Opened(&x, nil) + if err != nil { + t.Fatalf("opening x: %v", err) + } + err = reg.Seen(&y, nil) + if err == nil { + t.Fatalf("seeing unopened y: expected error, didn't get one") + } + err = reg.Seen(&x, nil) + if err != nil { + t.Fatalf("seeing opened x: unexpected error %v", err) + } + err = reg.Created(&y, nil) + if err != nil { + t.Fatalf("creating y: %v", err) + } + err = reg.Opened(&y, nil) + if err != nil { + t.Fatalf("opening y: %v", err) + } + err = reg.Closed(&x, nil) + if err != nil { + t.Fatalf("closing x: %v", err) + } + err = reg.Closed(&x, nil) + if err == nil { + t.Fatalf("double-closing x: expected error, didn't get one") + } + err = reg.Destroyed(&x, nil) + if err != nil { + t.Fatalf("destroying x: got unexpected error %v", err) + } + err, errs := auditor.FinalCheck() + if err == nil { + t.Fatalf("unclosed y not detected") + } + _ = errs +} + +type ignoreLiveness struct { + skippable *int +} + +func (ign *ignoreLiveness) Live(o interface{}, _ *testhook.RegistryEntry) error { + if ptr, ok := o.(*int); ok { + if ptr == ign.skippable { + return nil + } + } + return errors.New("unexpected live object") +} + +var _ testhook.RegistryHookLive = &ignoreLiveness{} + +func TestAuditor_DiscardError(t *testing.T) { + var x, y int + iptr := reflect.TypeOf(&x) + auditor := testhook.NewVerifyCloseAuditor(testhook.RegistryHooks{iptr: &ignoreLiveness{skippable: &y}}) + reg, err := auditor.Registry(&x) + if err != nil { + t.Fatalf("requesting registry: %v", err) + } + err = reg.Created(&x, nil) + if err != nil { + t.Fatalf("creating x: %v", err) + } + err = reg.Opened(&x, nil) + if err != nil { + t.Fatalf("opening x: %v", err) + } + err = reg.Seen(&y, nil) + if err == nil { + t.Fatalf("seeing unopened y: expected error, didn't get one") + } + err = reg.Seen(&x, nil) + if err != nil { + t.Fatalf("seeing opened x: unexpected error %v", err) + } + err = reg.Created(&y, nil) + if err != nil { + t.Fatalf("creating y: %v", err) + } + err = reg.Opened(&y, nil) + if err != nil { + t.Fatalf("opening y: %v", err) + } + err = reg.Closed(&x, nil) + if err != nil { + t.Fatalf("closing x: %v", err) + } + err = reg.Closed(&x, nil) + if err == nil { + t.Fatalf("double-closing x: expected error, didn't get one") + } + err = reg.Destroyed(&x, nil) + if err != nil { + t.Fatalf("destroying x: got unexpected error %v", err) + } + err, errs := auditor.FinalCheck() + if err != nil { + t.Fatalf("expected to skip y, instead got err %v, error list %v", err, errs) + } +} + +func TestAuditor_KeepError(t *testing.T) { + var x, y, z int + iptr := reflect.TypeOf(&x) + auditor := testhook.NewVerifyCloseAuditor(testhook.RegistryHooks{iptr: &ignoreLiveness{skippable: &z}}) + reg, err := auditor.Registry(&x) + if err != nil { + t.Fatalf("requesting registry: %v", err) + } + err = reg.Created(&x, nil) + if err != nil { + t.Fatalf("creating x: %v", err) + } + err = reg.Opened(&x, nil) + if err != nil { + t.Fatalf("opening x: %v", err) + } + err = reg.Seen(&y, nil) + if err == nil { + t.Fatalf("seeing unopened y: expected error, didn't get one") + } + err = reg.Seen(&x, nil) + if err != nil { + t.Fatalf("seeing opened x: unexpected error %v", err) + } + err = reg.Created(&y, nil) + if err != nil { + t.Fatalf("creating y: %v", err) + } + err = reg.Opened(&y, nil) + if err != nil { + t.Fatalf("opening y: %v", err) + } + err = reg.Closed(&x, nil) + if err != nil { + t.Fatalf("closing x: %v", err) + } + err = reg.Closed(&x, nil) + if err == nil { + t.Fatalf("double-closing x: expected error, didn't get one") + } + err = reg.Destroyed(&x, nil) + if err != nil { + t.Fatalf("destroying x: %v", err) + } + err, errs := auditor.FinalCheck() + if err == nil { + t.Fatalf("undestroyed y not detected") + } + _ = errs +} + +type failHook struct { + calls int +} + +func (f *failHook) Opened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error { + f.calls++ + return errors.New("failHook always fails") +} + +func (f *failHook) WasOpened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error { + f.calls++ + return errors.New("failHook always fails") +} + +// Implement WasSeen but not Seen, so we can verify that the only-one +// case works in both directions. +func (f *failHook) WasSeen(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error { + f.calls++ + return errors.New("failHook always fails") +} + +func (f *failHook) Closed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error { + f.calls++ + return errors.New("failHook always fails") +} + +func (f *failHook) WasClosed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error { + f.calls++ + return errors.New("failHook always fails") +} + +func (f *failHook) Live(i interface{}, ent *testhook.RegistryEntry) error { + f.calls++ + return errors.New("failHook always fails") +} + +type successHook struct { + calls int +} + +func (s *successHook) Opened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error { + s.calls++ + return nil +} + +func (s *successHook) WasOpened(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error { + s.calls++ + return nil +} + +func (s *successHook) Closed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error { + s.calls++ + return nil +} + +func (s *successHook) Seen(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry) error { + s.calls++ + return nil +} + +// successHook allows an error to leak from WasClosed. +func (s *successHook) WasClosed(i interface{}, kv testhook.KV, _ *testhook.RegistryEntry, err error) error { + s.calls++ + return err +} + +func (s *successHook) Live(i interface{}, ent *testhook.RegistryEntry) error { + s.calls++ + return errors.New("even successHook can fail sometimes") +} + +func TestComposedHooks(t *testing.T) { + s := &successHook{} + f := &failHook{} + sExp, fExp := 0, 0 + var err error + checkExp := func(when string) { + if s.calls != sExp { + t.Fatalf("after %s, expected %d calls to successHook, got %d", when, sExp, s.calls) + } + if f.calls != fExp { + t.Fatalf("after %s, expected %d calls to failHook, got %d", when, fExp, f.calls) + } + } + combined := testhook.Compose(s, f) + // Opened: we expect both opened calls to be hit, and the error from + // the second to come back. + err = combined.Opened(nil, nil, nil) + sExp++ + fExp++ + checkExp("opened") + if err == nil { + t.Fatalf("composed hook, opened: didn't error") + } + if err.Error() != "failHook always fails" { + t.Fatalf("composed hook, opened: expected failHook always fails, got %v", err) + } + + // WasOpened: we expect the error from failHook to be overridden. + err = combined.WasOpened(nil, nil, nil, nil) + sExp++ + fExp++ + checkExp("wasOpened") + if err != nil { + t.Fatalf("composed hook, wasOpened: expected no error, got %v", err) + } + + // Seen: nothing to call for failHook + err = combined.Seen(nil, nil, nil) + sExp++ + checkExp("seen") + if err != nil { + t.Fatalf("composed hook, seen: expected no error, got %v", err) + } + + // WasSeen: nothing to call for successHook + err = combined.WasSeen(nil, nil, nil, nil) + fExp++ + checkExp("wasSeen") + if err == nil { + t.Fatalf("composed hook, wasSeen: expected error, didn't get it") + } + + // WasClosed: expect both to get called, but successHook to leak the error up + err = combined.WasClosed(nil, nil, nil, nil) + sExp++ + fExp++ + checkExp("wasClosed") + if err == nil { + t.Fatalf("composed hook, wasClosed: expected error, didn't get it") + } + + // Live: the failure from successHook (oops) should prevent failHook from + // being called. + err = combined.Live(nil, nil) + sExp++ + checkExp("live") + if err == nil { + t.Fatalf("composed hook, live: expected error, didn't get it") + } +} diff --git a/testhook/cleanup1.13.go b/testhook/cleanup1.13.go new file mode 100644 index 000000000..05aeb48e7 --- /dev/null +++ b/testhook/cleanup1.13.go @@ -0,0 +1,47 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !go1.14 + +package testhook + +import ( + "sync" + "testing" +) + +var cleanupFuncs []func() +var cleanupMu sync.Mutex + +func init() { + RegisterPostTestHook(runCleanupFuncs) +} + +func runCleanupFuncs() error { + cleanupMu.Lock() + defer cleanupMu.Unlock() + for _, fn := range cleanupFuncs { + fn() + } + return nil +} + +// Cleanup in 1.13 logs a message about skipping a cleanup function, but +// allows things to build. Cleanup in 1.14 uses tb.Cleanup to register +// a cleanup function to call when a test completes. +func Cleanup(tb testing.TB, fn func()) { + cleanupMu.Lock() + defer cleanupMu.Unlock() + cleanupFuncs = append(cleanupFuncs, fn) +} diff --git a/testhook/cleanup1.14.go b/testhook/cleanup1.14.go new file mode 100644 index 000000000..9ca6024be --- /dev/null +++ b/testhook/cleanup1.14.go @@ -0,0 +1,28 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.14 + +package testhook + +import ( + "testing" +) + +// Cleanup in 1.13 logs a message about skipping a cleanup function, but +// allows things to build. Cleanup in 1.14 uses tb.Cleanup to register +// a cleanup function to call when a test completes. +func Cleanup(tb testing.TB, fn func()) { + tb.Cleanup(fn) +} diff --git a/testhook/hook.go b/testhook/hook.go new file mode 100644 index 000000000..8316d0806 --- /dev/null +++ b/testhook/hook.go @@ -0,0 +1,107 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testhook + +import ( + "fmt" + "io/ioutil" + "os" + "sync" + "testing" +) + +// Callback denotes a function which can be run on a testing.T, or testing.B, +// which performs additional functions typically before or after tests. +type Callback func() error + +var preHooks []Callback +var postHooks []Callback +var mu sync.Mutex + +// RegisterPostTestHook registers a function to be called after tests +// are run. It should return a nil error if it's okay, and a non-nil +// error to cause a non-zero exit status. +func RegisterPostTestHook(fn Callback) { + mu.Lock() + defer mu.Unlock() + postHooks = append(postHooks, fn) +} + +// RegisterPreTestHook registers a function to be called after tests +// are run. It should return a nil error if it's okay, and a non-nil +// error to cause a non-zero exit status. +func RegisterPreTestHook(fn Callback) { + mu.Lock() + defer mu.Unlock() + preHooks = append(preHooks, fn) +} + +// RunTestsWithHooks is a suitable implementation for TestMain; you can +// just invoke this from your TestMain, passing in m, and it runs the tests +// and then runs any registered pre/post hooks. If the hooks themselves try +// to register hooks, you will deadlock. Don't do that. +func RunTestsWithHooks(m *testing.M) { + var ret int + mu.Lock() + for _, fn := range preHooks { + err := fn() + if err != nil { + fmt.Fprintf(os.Stderr, "pre-hook failure: %v\n", err) + ret = 1 + } + } + mu.Unlock() + if ret != 0 { + fmt.Fprint(os.Stderr, "pre-hooks failed, aborting.\n") + os.Exit(ret) + } + ret = m.Run() + mu.Lock() + defer mu.Unlock() + for _, fn := range postHooks { + err := fn() + if err != nil { + fmt.Fprintf(os.Stderr, "post-hook failure: %v\n", err) + ret = 1 + } + } + os.Exit(ret) +} + +// TempDir creates a temp directory that will be automatically deleted when +// this test completes, using go1.14's [TB].Cleanup() if available. +func TempDir(tb testing.TB, pattern string) (path string, err error) { + path, err = ioutil.TempDir("", pattern) + if err == nil { + Cleanup(tb, func() { + os.RemoveAll(path) + }) + } + return path, err +} + +// TempDirInDir creates a temp directory that will be automatically deleted when +// this test completes, using go1.14's [TB].Cleanup(), but with a specified +// path instead of the default Go TMPDIR. Only some tests use this, which is +// possibly an error... +func TempDirInDir(tb testing.TB, dir string, pattern string) (path string, err error) { + path, err = ioutil.TempDir(dir, pattern) + if err == nil { + Cleanup(tb, func() { + os.RemoveAll(path) + }) + } + return path, err +} diff --git a/testhook/registry.go b/testhook/registry.go new file mode 100644 index 000000000..31bde6d5b --- /dev/null +++ b/testhook/registry.go @@ -0,0 +1,524 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testhook + +import ( + "fmt" + "reflect" + "runtime/debug" + "sync" + "time" +) + +// KV represents a key/value mapping. You don't need to use the type +// for anything, it's just to make typing easier. +type KV map[string]interface{} + +// Registry represents a set of known objects of a given common type. +// A typical implementation might maintain a map of objects it's seen which +// haven't been deleted. A minimal implementation just does nothing. +// +// A Registry may implement reference-count semantics, or may treat +// reopening of a still-open object as an error. +// +// All objects provided to a registry should have the same type. +type Registry interface { + Created(interface{}, KV) error // Object has been created, and must be destroyed later. + Opened(interface{}, KV) error // Object has been opened, and must be closed later. + Seen(interface{}, KV) error // Object has been interacted with, and should be between an open and a close. + Closed(interface{}, KV) error // Object has been closed, and must have been opened previously. + Destroyed(interface{}, KV) error // Object has been destroyed + Live() (map[interface{}]*RegistryEntry, error) // Currently live objects +} + +// RegistryHook is a generic interface, but any real implementation should +// implement at least one of RegistryHookPreOpen, RegistryHookPostOpen, +// etcetera. Pre-hooks are called before any error checks by the registry, +// post-hooks are called with after those error checks, and are passed the +// (possibly nil) error that would be returned at that point. If the post-hook +// overrides the error, the registry continues with operations as though it +// hadn't occurred, which may be a very bad idea. +type RegistryHook interface{} + +type RegistryHookPreCreate interface { + Created(interface{}, KV, *RegistryEntry) error +} + +type RegistryHookPostCreate interface { + WasCreated(interface{}, KV, *RegistryEntry, error) error +} + +type RegistryHookPreOpen interface { + Opened(interface{}, KV, *RegistryEntry) error +} + +type RegistryHookPostOpen interface { + WasOpened(interface{}, KV, *RegistryEntry, error) error +} + +type RegistryHookPreSee interface { + Seen(interface{}, KV, *RegistryEntry) error +} + +type RegistryHookPostSee interface { + WasSeen(interface{}, KV, *RegistryEntry, error) error +} + +type RegistryHookPreClose interface { + Closed(interface{}, KV, *RegistryEntry) error +} + +type RegistryHookPostClose interface { + WasClosed(interface{}, KV, *RegistryEntry, error) error +} + +type RegistryHookPreDestroy interface { + Destroyed(interface{}, KV, *RegistryEntry) error +} + +type RegistryHookPostDestroy interface { + WasDestroyed(interface{}, KV, *RegistryEntry, error) error +} + +// If a registry's hooks implement RegistryHookLive, Live() should +// return only those entries for which a non-nil error was returned, with the +// error inserted in the RegistryEntry. +type RegistryHookLive interface { + Live(interface{}, *RegistryEntry) error +} + +// RegistryHooks represents a set of registry hook values to use for +// registries, corresponding to different types. +type RegistryHooks map[reflect.Type]RegistryHook + +// RegistryEntry represents the data we might have about an entry. Every +// entry in it could be zero-valued in some implementations +type RegistryEntry struct { + Error error + Stack []byte + Stamp time.Time + Data KV + OpenCount int +} + +// NopRegistry doesn't do anything; it exists to fit the interface but not +// consume resources. +type NopRegistry struct{} + +var _ Registry = &NopRegistry{} + +func (*NopRegistry) Created(interface{}, KV) error { return nil } +func (*NopRegistry) Opened(interface{}, KV) error { return nil } +func (*NopRegistry) Seen(interface{}, KV) error { return nil } +func (*NopRegistry) Closed(interface{}, KV) error { return nil } +func (*NopRegistry) Destroyed(interface{}, KV) error { return nil } +func (*NopRegistry) Live() (map[interface{}]*RegistryEntry, error) { return nil, nil } + +func NewNopRegistry() *NopRegistry { + return &NopRegistry{} +} + +// SimpleRegistry asserts that objects are created, then opened, then +// possibly seen, then closed, then destroyed, and that they are not +// opened more than once at a time, or destroyed while open. As a +// convenience feature for users whose use cases might rely on this, +// it will actually accept a new item being opened without being +// previously created; to prevent this, use a PreOpen hook that checks +// for a nil *RegistryEntry. The default Live check will report only +// objects with an open count other than 0, but if you provide a Live +// hook, you can return errors for objects still existing. +type SimpleRegistry struct { + hooks RegistryHook + entries map[interface{}]*RegistryEntry + mu sync.Mutex +} + +var _ Registry = &SimpleRegistry{} + +func NewSimpleRegistry(hooks RegistryHook) *SimpleRegistry { + return &SimpleRegistry{entries: map[interface{}]*RegistryEntry{}, hooks: hooks} +} + +func (s *SimpleRegistry) Created(o interface{}, kv KV) (err error) { + s.mu.Lock() + defer s.mu.Unlock() + existing := s.entries[o] + if hook, ok := s.hooks.(RegistryHookPreCreate); ok { + if err := hook.Created(o, kv, existing); err != nil { + return err + } + } + if existing != nil { + err = fmt.Errorf("object %T:%v previously registered at %v", o, o, existing.Stamp) + } else { + existing = &RegistryEntry{ + Stack: debug.Stack(), + Data: kv, + Stamp: time.Now(), + } + } + if hook, ok := s.hooks.(RegistryHookPostCreate); ok { + err = hook.WasCreated(o, kv, existing, err) + } + if err != nil { + return err + } + // if you overrode the error, or there wasn't one and you didn't + // create one, we now stash the entry. + s.entries[o] = existing + return nil +} + +func (s *SimpleRegistry) Opened(o interface{}, kv KV) (err error) { + s.mu.Lock() + defer s.mu.Unlock() + existing := s.entries[o] + if hook, ok := s.hooks.(RegistryHookPreOpen); ok { + if err := hook.Opened(o, kv, existing); err != nil { + return err + } + } + if existing == nil { + existing = &RegistryEntry{ + Stack: debug.Stack(), + Data: kv, + Stamp: time.Now(), + } + s.entries[o] = existing + } + existing.OpenCount++ + if existing.OpenCount > 1 { + err = fmt.Errorf("object %T:%v opened %d times", o, o, existing.OpenCount) + } + if hook, ok := s.hooks.(RegistryHookPostOpen); ok { + err = hook.WasOpened(o, kv, existing, err) + } + if err != nil { + if existing != nil { + existing.OpenCount-- + } + return err + } + return nil +} + +func (s *SimpleRegistry) Seen(o interface{}, kv KV) (err error) { + s.mu.Lock() + defer s.mu.Unlock() + existing := s.entries[o] + if hook, ok := s.hooks.(RegistryHookPreSee); ok { + if err = hook.Seen(o, kv, existing); err != nil { + return err + } + } + if existing == nil { + err = fmt.Errorf("object %T:%v seen but not previously registered", o, o) + } + if hook, ok := s.hooks.(RegistryHookPostSee); ok { + err = hook.WasSeen(o, kv, existing, err) + } + return err +} + +func (s *SimpleRegistry) Closed(o interface{}, kv KV) (err error) { + s.mu.Lock() + defer s.mu.Unlock() + existing := s.entries[o] + if hook, ok := s.hooks.(RegistryHookPreClose); ok { + if err = hook.Closed(o, kv, existing); err != nil { + return err + } + } + if existing == nil { + err = fmt.Errorf("object %T:%v closed but not previously registered", o, o) + } else { + existing.OpenCount-- + if existing.OpenCount < 0 { + err = fmt.Errorf("object %T:%v closed more often than it was open: %d", o, o, existing.OpenCount) + } + } + if hook, ok := s.hooks.(RegistryHookPostClose); ok { + err = hook.WasClosed(o, kv, existing, err) + } + if err != nil { + // if a close "failed", we don't want to count it as being closed. + if existing != nil { + existing.OpenCount++ + } + return err + } + return nil +} + +func (s *SimpleRegistry) Destroyed(o interface{}, kv KV) (err error) { + s.mu.Lock() + defer s.mu.Unlock() + existing := s.entries[o] + if hook, ok := s.hooks.(RegistryHookPreDestroy); ok { + if err = hook.Destroyed(o, kv, existing); err != nil { + return err + } + } + if existing == nil { + err = fmt.Errorf("object %T:%v destroyed but not previously registered", o, o) + } else if existing.OpenCount != 0 { + err = fmt.Errorf("object %T:%v destroyed while open count is %d", o, o, existing.OpenCount) + } + if hook, ok := s.hooks.(RegistryHookPostDestroy); ok { + err = hook.WasDestroyed(o, kv, existing, err) + } + if err != nil { + fmt.Printf("destroyed error: %v\n", err) + return err + } + // remove the entry from the list. + delete(s.entries, o) + return nil +} + +func (s *SimpleRegistry) Live() (results map[interface{}]*RegistryEntry, err error) { + s.mu.Lock() + defer s.mu.Unlock() + results = make(map[interface{}]*RegistryEntry) + if hook, ok := s.hooks.(RegistryHookLive); ok { + for k, v := range s.entries { + v.Error = hook.Live(k, v) + if v.Error != nil { + results[k] = v + } + } + } else { + for k, v := range s.entries { + if v.OpenCount != 0 { + v.Error = fmt.Errorf("open count %d", v.OpenCount) + results[k] = v + } + } + } + return results, nil +} + +type PreHook func(interface{}, KV, *RegistryEntry) error +type PostHook func(interface{}, KV, *RegistryEntry, error) error +type LiveHook func(interface{}, *RegistryEntry) error + +// PluggableRegistry is a registry which takes dynamically-generated functions, +// and calls them if they're not nil. +type PluggableRegistry struct { + implCreated PreHook + implWasCreated PostHook + implOpened PreHook + implWasOpened PostHook + implSeen PreHook + implWasSeen PostHook + implClosed PreHook + implWasClosed PostHook + implDestroyed PreHook + implWasDestroyed PostHook + implLive LiveHook +} + +func (p *PluggableRegistry) Created(i interface{}, kv KV, ent *RegistryEntry) error { + if p.implCreated != nil { + return p.implCreated(i, kv, ent) + } + return nil +} + +func (p *PluggableRegistry) WasCreated(i interface{}, kv KV, ent *RegistryEntry, err error) error { + if p.implWasCreated != nil { + return p.implWasCreated(i, kv, ent, err) + } + return nil +} + +func (p *PluggableRegistry) Opened(i interface{}, kv KV, ent *RegistryEntry) error { + if p.implOpened != nil { + return p.implOpened(i, kv, ent) + } + return nil +} + +func (p *PluggableRegistry) WasOpened(i interface{}, kv KV, ent *RegistryEntry, err error) error { + if p.implWasOpened != nil { + return p.implWasOpened(i, kv, ent, err) + } + return nil +} + +func (p *PluggableRegistry) Seen(i interface{}, kv KV, ent *RegistryEntry) error { + if p.implSeen != nil { + return p.implSeen(i, kv, ent) + } + return nil +} + +func (p *PluggableRegistry) WasSeen(i interface{}, kv KV, ent *RegistryEntry, err error) error { + if p.implWasSeen != nil { + return p.implWasSeen(i, kv, ent, err) + } + return nil +} + +func (p *PluggableRegistry) Closed(i interface{}, kv KV, ent *RegistryEntry) error { + if p.implClosed != nil { + return p.implClosed(i, kv, ent) + } + return nil +} + +func (p *PluggableRegistry) WasClosed(i interface{}, kv KV, ent *RegistryEntry, err error) error { + if p.implWasClosed != nil { + return p.implWasClosed(i, kv, ent, err) + } + return nil +} + +func (p *PluggableRegistry) Destroyed(i interface{}, kv KV, ent *RegistryEntry) error { + if p.implDestroyed != nil { + return p.implDestroyed(i, kv, ent) + } + return nil +} + +func (p *PluggableRegistry) WasDestroyed(i interface{}, kv KV, ent *RegistryEntry, err error) error { + if p.implWasDestroyed != nil { + return p.implWasDestroyed(i, kv, ent, err) + } + return nil +} + +func (p *PluggableRegistry) Live(i interface{}, ent *RegistryEntry) error { + if p.implLive != nil { + return p.implLive(i, ent) + } + return nil +} + +func composePreHooks(fns ...PreHook) PreHook { + if len(fns) == 0 { + return nil + } + if len(fns) == 1 { + return fns[0] + } + return func(i interface{}, kv KV, ent *RegistryEntry) error { + for _, fn := range fns { + err := fn(i, kv, ent) + if err != nil { + return err + } + } + return nil + } +} + +func composePostHooks(fns ...PostHook) PostHook { + if len(fns) == 0 { + return nil + } + if len(fns) == 1 { + return fns[0] + } + return func(i interface{}, kv KV, ent *RegistryEntry, err error) error { + // We run the functions in reverse order, so the last + // hook added has the option of overriding a lower hook's + // opinion. + for i := range fns { + fn := fns[len(fns)-1-i] + err = fn(i, kv, ent, err) + } + return err + } +} + +func composeLiveHooks(fns ...LiveHook) LiveHook { + if len(fns) == 0 { + return nil + } + if len(fns) == 1 { + return fns[0] + } + return func(i interface{}, ent *RegistryEntry) error { + for _, fn := range fns { + err := fn(i, ent) + if err != nil { + return err + } + } + return nil + } +} + +// Compose takes a list of RegistryHook objects, and combines their hook +// functions into a unified registry. For Pre hooks and Live hooks, the +// functions are called in the order they were provided to this function, +// and the first one to return an error causes the remainder to be +// skipped; for Post hooks, they are called in the opposite order, and +// the whole list continues to be called, with each function getting passed +// the error (or non-error) value returned by the previous one. +func Compose(hooks ...RegistryHook) *PluggableRegistry { + var created, opened, seen, closed, destroyed []PreHook + var wasCreated, wasOpened, wasSeen, wasClosed, wasDestroyed []PostHook + var live []LiveHook + for _, h := range hooks { + if hook, ok := h.(RegistryHookPreCreate); ok { + created = append(created, hook.Created) + } + if hook, ok := h.(RegistryHookPostCreate); ok { + wasCreated = append(wasCreated, hook.WasCreated) + } + if hook, ok := h.(RegistryHookPreOpen); ok { + opened = append(opened, hook.Opened) + } + if hook, ok := h.(RegistryHookPostOpen); ok { + wasOpened = append(wasOpened, hook.WasOpened) + } + if hook, ok := h.(RegistryHookPreSee); ok { + seen = append(seen, hook.Seen) + } + if hook, ok := h.(RegistryHookPostSee); ok { + wasSeen = append(wasSeen, hook.WasSeen) + } + if hook, ok := h.(RegistryHookPreClose); ok { + closed = append(closed, hook.Closed) + } + if hook, ok := h.(RegistryHookPostClose); ok { + wasClosed = append(wasClosed, hook.WasClosed) + } + if hook, ok := h.(RegistryHookPreDestroy); ok { + destroyed = append(destroyed, hook.Destroyed) + } + if hook, ok := h.(RegistryHookPostDestroy); ok { + wasDestroyed = append(wasDestroyed, hook.WasDestroyed) + } + if hook, ok := h.(RegistryHookLive); ok { + live = append(live, hook.Live) + } + } + return &PluggableRegistry{ + implCreated: composePreHooks(created...), + implWasCreated: composePostHooks(wasCreated...), + implOpened: composePreHooks(opened...), + implWasOpened: composePostHooks(wasOpened...), + implSeen: composePreHooks(seen...), + implWasSeen: composePostHooks(wasSeen...), + implClosed: composePreHooks(closed...), + implWasClosed: composePostHooks(wasClosed...), + implDestroyed: composePreHooks(destroyed...), + implWasDestroyed: composePostHooks(wasDestroyed...), + implLive: composeLiveHooks(live...), + } +} diff --git a/translator_test.go b/translator_test.go index dab23bbd2..7eaf62859 100644 --- a/translator_test.go +++ b/translator_test.go @@ -227,11 +227,12 @@ func TestTranslation_Reset(t *testing.T) { pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), )}, ) + defer c.Close() - node0 := c[0] - node1 := c[1] - node2 := c[2] - node3 := c[3] + node0 := c.GetNode(0) + node1 := c.GetNode(1) + node2 := c.GetNode(2) + node3 := c.GetNode(3) ctx := context.Background() idx := "i" @@ -321,9 +322,10 @@ func TestTranslation_Replication(t *testing.T) { pilosa.OptServerReplicaN(2), )}, ) + defer c.Close() - node0 := c[0] - node1 := c[1] + node0 := c.GetNode(0) + node1 := c.GetNode(1) ctx := context.Background() idx := "i" @@ -362,7 +364,7 @@ func TestTranslation_Replication(t *testing.T) { node0.QueryExpect(t, idx, "", `Row(f=1)`, exp) // Kill one node. - if err := node1.Command.Close(); err != nil { + if err := c.CloseAndRemove(1); err != nil { t.Fatal(err) } @@ -396,8 +398,8 @@ func TestTranslation_Coordinator(t *testing.T) { ) defer c.Close() - node0 := c[0] - node1 := c[1] + node0 := c.GetNode(0) + node1 := c.GetNode(1) ctx := context.Background() idx := "i" diff --git a/tx_test.go b/tx_test.go index ae5e0b984..4fbc5b8b3 100644 --- a/tx_test.go +++ b/tx_test.go @@ -58,6 +58,7 @@ func queryBalances(m0api *pilosa.API, acctOwnerID uint64, fldAcct0, fldAcct1, in acct1bal = mustQueryAcct(m0api, acctOwnerID, fldAcct1, index) return } + func skipForRoaring(t *testing.T) { src := os.Getenv("PILOSA_TXSRC") // once txfactory.go DefaultTxsrc != RoaringTxn, this @@ -67,7 +68,7 @@ func skipForRoaring(t *testing.T) { } } -func TestAPI_ImportAIR(t *testing.T) { +func TestAPI_ImportAtomicRecord(t *testing.T) { skipForRoaring(t) c := test.MustRunCluster(t, 1, []server.CommandOption{ @@ -79,7 +80,7 @@ func TestAPI_ImportAIR(t *testing.T) { ) defer c.Close() - m0 := c[0] + m0 := c.GetNode(0) m0api := m0.API ctx := context.Background() diff --git a/utils_internal_test.go b/utils_internal_test.go index f24fded67..7ba8c45d4 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -19,15 +19,17 @@ import ( "io/ioutil" "path/filepath" "sync" + "testing" "time" "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" ) // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. -func NewTestCluster(n int) *cluster { - path, err := ioutil.TempDir("", "pilosa-cluster-") +func NewTestCluster(tb testing.TB, n int) *cluster { + path, err := testhook.TempDir(tb, "pilosa-cluster-") if err != nil { panic(err) } @@ -88,6 +90,7 @@ type ClusterCluster struct { mu sync.RWMutex resizing bool resizeDone chan struct{} + tb testing.TB } type commonClusterSettings struct { @@ -226,7 +229,7 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) t.common.Nodes = append(t.common.Nodes, node) // create node-specific temp directory - path, err := ioutil.TempDir(*TempDir, fmt.Sprintf("pilosa-cluster-node-%d-", i)) + path, err := testhook.TempDirInDir(t.tb, *TempDir, fmt.Sprintf("pilosa-cluster-node-%d-", i)) if err != nil { return nil, err } @@ -262,10 +265,11 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) } // NewClusterCluster returns a new instance of test.Cluster. -func NewClusterCluster(n int) *ClusterCluster { +func NewClusterCluster(tb testing.TB, n int) *ClusterCluster { tc := &ClusterCluster{ common: &commonClusterSettings{}, + tb: tb, } // add clusters diff --git a/view.go b/view.go index 92c5a60df..3acd3615a 100644 --- a/view.go +++ b/view.go @@ -29,6 +29,7 @@ import ( "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" + "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -158,6 +159,7 @@ func (v *view) open() error { return err } + _ = testhook.Opened(v.holder.Auditor, v, nil) v.holder.Logger.Debugf("successfully opened index/field/view: %s/%s/%s", v.index, v.field, v.name) return nil } @@ -224,6 +226,9 @@ shardLoop: func (v *view) close() error { v.mu.Lock() defer v.mu.Unlock() + defer func() { + _ = testhook.Closed(v.holder.Auditor, v, nil) + }() // Close all fragments. eg, ctx := errgroup.WithContext(context.Background()) @@ -394,6 +399,7 @@ func (v *view) deleteFragment(shard uint64) error { v.holder.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard) idx := f.holder.Index(v.index) + f.Close() if err := idx.Txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil { return errors.Wrap(err, "DeleteFragment") } diff --git a/view_internal_test.go b/view_internal_test.go index bcde5f871..c4f39bc48 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -15,16 +15,16 @@ package pilosa import ( - "io/ioutil" "testing" "time" + "github.com/pilosa/pilosa/v2/testhook" "golang.org/x/sync/errgroup" ) // mustOpenView returns a new instance of View with a temporary path. -func mustOpenView(index, field, name string) *view { - path, err := ioutil.TempDir(*TempDir, "pilosa-view-") +func mustOpenView(tb testing.TB, index, field, name string) *view { + path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-view-") if err != nil { panic(err) } @@ -38,7 +38,9 @@ func mustOpenView(index, field, name string) *view { h.Path = path // h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment idx, err := h.createIndex(index, IndexOptions{}) - _ = idx + testhook.Cleanup(tb, func() { + h.Close() + }) panicOn(err) v := newView(h, path, index, field, name, fo) @@ -54,7 +56,7 @@ func mustOpenView(index, field, name string) *view { // Ensure view can open and retrieve a fragment. func TestView_DeleteFragment(t *testing.T) { - v := mustOpenView("i", "f", "v") + v := mustOpenView(t, "i", "f", "v") defer v.close() shard := uint64(9) @@ -89,7 +91,7 @@ func TestView_DeleteFragment(t *testing.T) { // if the broadcast operation takes a bit of time. func TestView_CreateFragmentRace(t *testing.T) { var creates errgroup.Group - v := mustOpenView("i", "f", "v") + v := mustOpenView(t, "i", "f", "v") defer v.close() // Use a broadcaster which intentionally fails.