scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-10 15:01:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1425 from codysoyland/enhance-test-utilities

Browse source 
Enhance test utilities
This commit is contained in:
Cody Soyland 2018-06-28 13:45:39 -05:00 • committed by GitHub
commit 112e8e68b6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 141 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ctl/export_test.go
View file

@ -44,7 +44,7 @@ func TestExportCommand_Validation(t *testing.T) {
}
func TestExportCommand_Run(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)

8
ctl/import_test.go
View file

@ -61,7 +61,7 @@ func TestImportCommand_Run(t *testing.T) {
t.Fatal(err)
}
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
cm.Index = "i"
@ -86,7 +86,7 @@ func TestImportCommand_RunValue(t *testing.T) {
t.Fatal(err)
}
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
@ -102,7 +102,7 @@ func TestImportCommand_RunValue(t *testing.T) {
}
func TestImportCommand_InvalidFile(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
@ -176,7 +176,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
}
func TestImportCommand_BugOverwriteValue(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)

4
executor_test.go
View file

@ -267,7 +267,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
// Ensure a set query can be executed.
func TestExecutor_Execute_SetBit(t *testing.T) {
t.Run("ID", func(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
hldr.SetBit("i", "f", 1, 0)
@ -312,7 +312,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
})
t.Run("Keys", func(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})

6
http/client_test.go
View file

@ -219,7 +219,7 @@ func TestClient_MultiNode(t *testing.T) {
// Ensure client can bulk import data.
func TestClient_Import(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
host := cmd.URL()
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
@ -249,7 +249,7 @@ func TestClient_Import(t *testing.T) {
// Ensure client can bulk import value data.
func TestClient_ImportValue(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
host := cmd.URL()
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
@ -321,7 +321,7 @@ func TestClient_ImportValue(t *testing.T) {
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
func TestClient_FragmentBlocks(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}

6
http/translator_test.go
View file

@ -70,7 +70,7 @@ func TestTranslateStore_Reader(t *testing.T) {
}
opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0]
main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
defer main.Close()
@ -121,7 +121,7 @@ func TestTranslateStore_Reader(t *testing.T) {
}
opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0]
main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
defer main.Close()
defer close(done)
@ -152,7 +152,7 @@ func TestTranslateStore_Reader(t *testing.T) {
}
opts := server.OptCommandServerOptions(pilosa.OptServerPrimaryTranslateStore(translateStore))
main := test.MustRunMainWithCluster(t, 1, []server.CommandOption{opts})[0]
main := test.MustRunCluster(t, 1, []server.CommandOption{opts})[0]
defer main.Close()
_, err := http.NewTranslateStore(main.Server.URI.String()).Reader(context.Background(), 0)

30
server/cluster_test.go
View file

@ -31,7 +31,7 @@ import (
// Ensure program can send/receive broadcast messages.
func TestMain_SendReceiveMessage(t *testing.T) {
ms := test.MustRunMainWithCluster(t, 2)
ms := test.MustRunCluster(t, 2)
m0, m1 := ms[0], ms[1]
defer m0.Close()
defer m1.Close()
@ -116,7 +116,7 @@ func TestMain_SendReceiveMessage(t *testing.T) {
// Ensure that an empty node comes up in a NORMAL state.
func TestClusterResize_EmptyNode(t *testing.T) {
m0 := test.MustRunMain()
m0 := test.MustRunCommand()
defer m0.Close()
if m0.API.State() != pilosa.ClusterStateNormal {
@ -126,7 +126,7 @@ func TestClusterResize_EmptyNode(t *testing.T) {
// Ensure that a cluster of empty nodes comes up in a NORMAL state.
func TestClusterResize_EmptyNodes(t *testing.T) {
clus := test.MustRunMainWithCluster(t, 2)
clus := test.MustRunCluster(t, 2)
defer clus[0].Close()
defer clus[1].Close()
@ -140,7 +140,7 @@ func TestClusterResize_EmptyNodes(t *testing.T) {
// Ensure that adding a node correctly resizes the cluster.
func TestClusterResize_AddNode(t *testing.T) {
t.Run("NoData", func(t *testing.T) {
clus := test.MustRunMainWithCluster(t, 2)
clus := test.MustRunCluster(t, 2)
if !checkClusterState(clus[0], pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State())
@ -150,7 +150,7 @@ func TestClusterResize_AddNode(t *testing.T) {
})
t.Run("WithIndex", func(t *testing.T) {
// Configure node0
m0 := test.MustRunMainWithCluster(t, 1)[0]
m0 := test.MustRunCluster(t, 1)[0]
defer m0.Close()
seed := m0.GossipAddress()
@ -166,7 +166,7 @@ func TestClusterResize_AddNode(t *testing.T) {
}
// Configure node1
m1 := test.NewMainWithCluster(false)
m1 := test.NewCommandNode(false)
m1.Config.Gossip.Port = "0"
m1.Config.Gossip.Seeds = []string{seed}
err := m1.Start()
@ -183,7 +183,7 @@ func TestClusterResize_AddNode(t *testing.T) {
})
t.Run("ContinuousSlices", func(t *testing.T) {
// Configure node0
m0 := test.MustRunMainWithCluster(t, 1)[0]
m0 := test.MustRunCluster(t, 1)[0]
defer m0.Close()
seed := m0.GossipAddress()
@ -207,7 +207,7 @@ func TestClusterResize_AddNode(t *testing.T) {
}
// Configure node1
m1 := test.NewMainWithCluster(false)
m1 := test.NewCommandNode(false)
m1.Config.Gossip.Port = "0"
m1.Config.Gossip.Seeds = []string{seed}
err := m1.Start()
@ -224,7 +224,7 @@ func TestClusterResize_AddNode(t *testing.T) {
})
t.Run("SkippedSlice", func(t *testing.T) {
// Configure node0
m0 := test.MustRunMainWithCluster(t, 1)[0]
m0 := test.MustRunCluster(t, 1)[0]
defer m0.Close()
seed := m0.GossipAddress()
@ -248,7 +248,7 @@ func TestClusterResize_AddNode(t *testing.T) {
}
// Configure node1
m1 := test.NewMainWithCluster(false)
m1 := test.NewCommandNode(false)
m1.Config.Gossip.Port = "0"
m1.Config.Gossip.Seeds = []string{seed}
err := m1.Start()
@ -269,7 +269,7 @@ func TestClusterResize_AddNode(t *testing.T) {
func TestCluster_GossipMembership(t *testing.T) {
t.Run("Node0Down", func(t *testing.T) {
// Configure node0
m0 := test.MustRunMainWithCluster(t, 1)[0]
m0 := test.MustRunCluster(t, 1)[0]
defer m0.Close()
seed := m0.GossipAddress()
@ -277,7 +277,7 @@ func TestCluster_GossipMembership(t *testing.T) {
var eg errgroup.Group
// Configure node1
m1 := test.NewMainWithCluster(false)
m1 := test.NewCommandNode(false)
defer m1.Close()
eg.Go(func() error {
m1.Config.Gossip.Port = "0"
@ -291,7 +291,7 @@ func TestCluster_GossipMembership(t *testing.T) {
})
// Configure node1
m2 := test.NewMainWithCluster(false)
m2 := test.NewCommandNode(false)
defer m2.Close()
eg.Go(func() error {
m2.Config.Gossip.Port = "0"
@ -324,7 +324,7 @@ func TestCluster_GossipMembership(t *testing.T) {
}
func TestClusterResize_RemoveNode(t *testing.T) {
cluster := test.MustRunMainWithCluster(t, 3)
cluster := test.MustRunCluster(t, 3)
m0 := cluster[0]
m1 := cluster[1]
@ -410,7 +410,7 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// checkClusterState polls a given cluster for its state until it
// receives a matching state. It polls up to n times before returning.
func checkClusterState(m *test.Main, state string, n int) bool {
func checkClusterState(m *test.Command, state string, n int) bool {
for i := 0; i < n; i++ {
if m.API.State() == state {
return true

4
server/handler_test.go
View file

@ -37,7 +37,7 @@ import (
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_Endpoints(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
h := cmd.Handler.(*http.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
@ -566,7 +566,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode)
}
clus := test.MustRunMainWithCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})})
clus := test.MustRunCluster(t, 1, []server.CommandOption{test.OptAllowedOrigins([]string{"http://test/"})})
w = httptest.NewRecorder()
h := clus[0].Handler.(*http.Handler).Handler
h.ServeHTTP(w, req)

8
server/server_test.go
View file

@ -40,7 +40,7 @@ func TestMain_Set_Quick(t *testing.T) {
}
if err := quick.Check(func(cmds []SetCommand) bool {
m := test.MustRunMain()
m := test.MustRunCommand()
defer m.Close()
// Create client.
@ -116,7 +116,7 @@ func TestMain_Set_Quick(t *testing.T) {
// Ensure program can set row attributes and retrieve them.
func TestMain_SetRowAttrs(t *testing.T) {
m := test.MustRunMain()
m := test.MustRunCommand()
defer m.Close()
// Create fields.
@ -193,7 +193,7 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Ensure program can set column attributes and retrieve them.
func TestMain_SetColumnAttrs(t *testing.T) {
m := test.MustRunMain()
m := test.MustRunCommand()
defer m.Close()
// Create fields.
@ -264,7 +264,7 @@ func tempMkdir(t *testing.T) string {
func TestMain_RecalculateHashes(t *testing.T) {
const clusterSize = 5
cluster := test.MustRunMainWithCluster(t, clusterSize)
cluster := test.MustRunCluster(t, clusterSize)
// Create the schema.
client0 := cluster[0].Client()

2
server_test.go
View file

@ -28,7 +28,7 @@ import (
// pilosa.Server was not having its remoteClient field set by an option and so
// it was using a nil client in monitorAntiEntropy.
func TestMonitorAntiEntropy(t *testing.T) {
cluster := test.MustRunMainWithCluster(t, 3, []server.CommandOption{test.OptAntiEntropyInterval(time.Millisecond * 20)})
cluster := test.MustRunCluster(t, 3, []server.CommandOption{test.OptAntiEntropyInterval(time.Millisecond * 20)})
client := cluster[1].Client()
err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{})
if err != nil {

2
stats_test.go
View file

@ -209,7 +209,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
}
func TestStatsCount_APICalls(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
cmd := test.MustRunCluster(t, 1)[0]
h := cmd.Handler.(*http.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}

168
test/pilosa.go
View file

@ -32,8 +32,8 @@ import (
)
////////////////////////////////////////////////////////////////////////////////////
// Main represents a test wrapper for server.Command.
type Main struct {
// Command represents a test wrapper for server.Command.
type Command struct {
*server.Command
commandOptions []server.CommandOption
@ -57,21 +57,14 @@ func OptAllowedOrigins(origins []string) server.CommandOption {
}
}
// GossipAddress returns the address on which gossip is listening after a Main
// has been setup. Useful to pass as a seed to other nodes when creating and
// testing clusters.
func (m *Main) GossipAddress() string {
return m.GossipTransport().URI.String()
}
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain(opts ...server.CommandOption) *Main {
// NewCommand returns a new instance of Main with a temporary data directory and random port.
func NewCommand(opts ...server.CommandOption) *Command {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts}
m := &Command{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr, opts...), commandOptions: opts}
m.Config.DataDir = path
m.Config.Bind = "http://localhost:0"
m.Config.Cluster.Disabled = true
@ -92,58 +85,17 @@ func NewMain(opts ...server.CommandOption) *Main {
return m
}
// NewMainWithCluster returns a new instance of Main with clustering enabled.
func NewMainWithCluster(isCoordinator bool, opts ...server.CommandOption) *Main {
m := NewMain(opts...)
// NewCommandNode returns a new instance of Command with clustering enabled.
func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
m := NewCommand(opts...)
m.Config.Cluster.Disabled = false
m.Config.Cluster.Coordinator = isCoordinator
return m
}
// MustRunMainWithCluster ruturns a running array of *Main where
// all nodes are joined via memberlist (i.e. clustering enabled).
func MustRunMainWithCluster(t *testing.T, size int, opts ...[]server.CommandOption) []*Main {
ma, err := runMainWithCluster(size, opts...)
if err != nil {
t.Fatalf("new main array with cluster: %v", err)
}
return ma
}
// runMainWithCluster runs an array of *Main where all nodes are
// joined via memberlist (i.e. clustering enabled).
func runMainWithCluster(size int, opts ...[]server.CommandOption) ([]*Main, error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
}
if len(opts) != size && len(opts) != 0 && len(opts) != 1 {
return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
}
mains := make([]*Main, size)
var gossipSeeds = make([]string, size)
for i := 0; i < size; i++ {
var commandOpts []server.CommandOption
if len(opts) > 0 {
commandOpts = opts[i%len(opts)]
}
m := NewMainWithCluster(i == 0, commandOpts...)
m.Config.Gossip.Port = "0"
m.Config.Gossip.Seeds = gossipSeeds[:i]
if err := m.Start(); err != nil {
return nil, errors.Wrapf(err, "Starting server %d", i)
}
gossipSeeds[i] = m.GossipTransport().URI.String()
mains[i] = m
}
return mains, nil
}
// MustRunMain returns a new, running Main. Panic on error.
func MustRunMain() *Main {
m := NewMain()
// MustRunCommand returns a new, running Main. Panic on error.
func MustRunCommand() *Command {
m := NewCommand()
m.Config.Metric.Diagnostics = false // Disable diagnostics.
if err := m.Start(); err != nil {
panic(err)
@ -151,14 +103,21 @@ func MustRunMain() *Main {
return m
}
// GossipAddress returns the address on which gossip is listening after a Main
// has been setup. Useful to pass as a seed to other nodes when creating and
// testing clusters.
func (m *Command) GossipAddress() string {
return m.GossipTransport().URI.String()
}
// Close closes the program and removes the underlying data directory.
func (m *Main) Close() error {
func (m *Command) Close() error {
defer os.RemoveAll(m.Config.DataDir)
return m.Command.Close()
}
// Reopen closes the program and reopens it.
func (m *Main) Reopen() error {
func (m *Command) Reopen() error {
if err := m.Command.Close(); err != nil {
return err
}
@ -180,10 +139,10 @@ func (m *Main) Reopen() error {
}
// URL returns the base URL string for accessing the running program.
func (m *Main) URL() string { return m.Server.URI.String() }
func (m *Command) URL() string { return m.Server.URI.String() }
// Client returns a client to connect to the program.
func (m *Main) Client() *http.InternalClient {
func (m *Command) Client() *http.InternalClient {
client, err := http.NewInternalClient(m.Server.URI.HostPort(), http.GetHTTPClient(nil))
if err != nil {
panic(err)
@ -192,7 +151,7 @@ func (m *Main) Client() *http.InternalClient {
}
// Query executes a query against the program through the HTTP API.
func (m *Main) Query(index, rawQuery, query string) (string, error) {
func (m *Command) Query(index, rawQuery, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query)
if resp.StatusCode != gohttp.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
@ -200,7 +159,7 @@ func (m *Main) Query(index, rawQuery, query string) (string, error) {
return resp.Body, nil
}
func (m *Main) RecalculateCaches() error {
func (m *Command) RecalculateCaches() error {
resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "")
if resp.StatusCode != 204 {
return fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
@ -208,6 +167,85 @@ func (m *Main) RecalculateCaches() error {
return nil
}
// Cluster represents a Pilosa cluster (multiple Command instances)
type Cluster []*Command
// Start runs a Cluster
func (c Cluster) Start() error {
var gossipSeeds = make([]string, len(c))
for i, cc := range c {
cc.Config.Gossip.Port = "0"
cc.Config.Gossip.Seeds = gossipSeeds[:i]
if err := cc.Start(); err != nil {
return errors.Wrapf(err, "starting server %d", i)
}
gossipSeeds[i] = cc.GossipAddress()
}
return nil
}
// Stop stops a Cluster
func (c Cluster) Close() error {
for i, cc := range c {
if err := cc.Close(); err != nil {
return errors.Wrapf(err, "stopping server %d", i)
}
}
return nil
}
// MustNewCluster creates a new cluster
func MustNewCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster {
c, err := newCluster(size, opts...)
if err != nil {
t.Fatalf("new cluster: %v", err)
}
return c
}
// newCluster creates a new cluster
func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
if size == 0 {
return nil, errors.New("cluster must contain at least one node")
}
if len(opts) != size && len(opts) != 0 && len(opts) != 1 {
return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
}
cluster := make(Cluster, size)
for i := 0; i < size; i++ {
var commandOpts []server.CommandOption
if len(opts) > 0 {
commandOpts = opts[i%len(opts)]
}
m := NewCommandNode(i == 0, commandOpts...)
cluster[i] = m
}
return cluster, nil
}
// runCluster creates and starts a new cluster
func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
cluster, err := newCluster(size, opts...)
if err != nil {
return nil, errors.Wrap(err, "new cluster")
}
if err = cluster.Start(); err != nil {
return nil, errors.Wrap(err, "starting cluster")
}
return cluster, nil
}
// MustRunCluster creates and starts a new cluster
func MustRunCluster(t *testing.T, size int, opts ...[]server.CommandOption) Cluster {
c, err := runCluster(size, opts...)
if err != nil {
t.Fatalf("run cluster: %v", err)
}
return c
}
////////////////////////////////////////////////////////////////////////////////////
// MustDo executes http.Do() with an http.NewRequest(). Panic on error.

4
test/pilosa_test.go
View file

@ -27,7 +27,7 @@ import (
func TestNewCluster(t *testing.T) {
numNodes := 3
cluster := test.MustRunMainWithCluster(t, numNodes)
cluster := test.MustRunCluster(t, numNodes)
coordinator := getCoordinator(cluster[0])
for i := 1; i < numNodes; i++ {
@ -78,7 +78,7 @@ func TestNewCluster(t *testing.T) {
}
}
func getCoordinator(m *test.Main) string {
func getCoordinator(m *test.Command) string {
hosts := m.API.Hosts(context.Background())
for _, host := range hosts {
if host.IsCoordinator {