Merge pull request #1323 from codysoyland/release-v0.10.1

Release v0.10.1

CHANGELOG.md

@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v0.10.1] - 2018-05-22
+
+This version contains 10 contribution from 4 contributors. There are 5 files changed, 222 insertions, and 54 deletions.
+
+### Added
+
+- Add Docker cluster/swarm tutorials ([#1296](https://github.com/pilosa/pilosa/pull/1296), [#1305](https://github.com/pilosa/pilosa/pull/1305))
+
+### Fixed
+
+- Use errors.Cause in handler so that we return correct status codes ([#1304](https://github.com/pilosa/pilosa/pull/1304), [#1309](https://github.com/pilosa/pilosa/pull/1309))
+- Close HTTP handler gracefully ([#1310](https://github.com/pilosa/pilosa/pull/1310))
+- Use lazy assignment for VERSION_ID so enterprise flag is set appropriately ([#1308](https://github.com/pilosa/pilosa/pull/1308))
+- Adjust cluster state tests so they aren't so dependent upon a sleep ([#1306](https://github.com/pilosa/pilosa/pull/1306))
+- Makefile enterprise build fixes ([#1301](https://github.com/pilosa/pilosa/pull/1301))
+- Re-add unintentionally removed check-clean ([#1295](https://github.com/pilosa/pilosa/pull/1295))
+- Fix syntax error and add i386 enterprise build ([#1293](https://github.com/pilosa/pilosa/pull/1293))
+
 ## [v0.10.0] - 2018-05-15
 
 This version contains 93 contribution from 8 contributors. There are 93 files changed, 4,495 insertions, and 5,392 deletions.

Makefile

@@ -2,7 +2,7 @@
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-VERSION_ID := $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
+VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
 BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
 BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)

docs/installation.md

@@ -42,7 +42,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v0.10.0-64-gf053d9a5
+    Version: v0.10.1-64-gf053d9a5
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:
@@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
 
 1. Download the latest release:
     ```
-    curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.10.0/pilosa-v0.10.0-darwin-amd64.tar.gz
+    curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.10.1/pilosa-v0.10.1-darwin-amd64.tar.gz
     ```
 
     Other releases can be downloaded from our Releases page on Github.
 
 2. Extract the binary:
     ```
-    tar xfz pilosa-v0.10.0-darwin-amd64.tar.gz
+    tar xfz pilosa-v0.10.1-darwin-amd64.tar.gz
     ```
 
 3. Move the binary into your PATH so you can run `pilosa` from any shell:
     ```
-    cp -i pilosa-v0.10.0-darwin-amd64/pilosa /usr/local/bin
+    cp -i pilosa-v0.10.1-darwin-amd64/pilosa /usr/local/bin
     ```
 
 4. Make sure Pilosa is installed successfully:
@@ -103,7 +103,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v0.10.0-64-gf053d9a5
+    Version: v0.10.1-64-gf053d9a5
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:
@@ -170,7 +170,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v0.10.0-64-gf053d9a5
+    Version: v0.10.1-64-gf053d9a5
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:
@@ -232,19 +232,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
 
 1. To install the latest version of Pilosa, download the latest release:
     ```
-    curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.10.0/pilosa-v0.10.0-linux-amd64.tar.gz
+    curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.10.1/pilosa-v0.10.1-linux-amd64.tar.gz
     ```
 
     Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github.
 
 2. Extract the binary:
     ```
-    tar xfz pilosa-v0.10.0-linux-amd64.tar.gz
+    tar xfz pilosa-v0.10.1-linux-amd64.tar.gz
     ```
 
 3. Move the binary into your PATH so you can run `pilosa` from any shell:
     ```
-    cp -i pilosa-v0.10.0-linux-amd64/pilosa /usr/local/bin
+    cp -i pilosa-v0.10.1-linux-amd64/pilosa /usr/local/bin
     ```
 
 4. Make sure Pilosa is installed successfully:
@@ -261,7 +261,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v0.10.0-64-gf053d9a5
+    Version: v0.10.1-64-gf053d9a5
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:
@@ -328,7 +328,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
     backing up, and more. Complete documentation is available
     at https://www.pilosa.com/docs/.
 
-    Version: v0.10.0-64-gf053d9a5
+    Version: v0.10.1-64-gf053d9a5
     Build Time: 2018-05-14T22:14:01+0000
 
     Usage:

docs/tutorials.md

@@ -3,6 +3,7 @@ title = "Tutorials"
 weight = 4
 nav = [
     "Setting Up a Secure Cluster",
+    "Setting Up a Docker Cluster",
     "Using Integer Field Values",
     "Storing Row and Column Attributes",
 ]
@@ -238,6 +239,161 @@ curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap
 
 Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa.
 
+### Setting Up a Docker Cluster
+
+In this tutorial, we will be setting up a 2-node Pilosa cluster using Docker containers.
+
+#### Running a Docker Cluster on a Single Server
+
+The instructions below require Docker 1.13 or better.
+
+Let's first be sure that the Pilosa image is up to date:
+```
+docker pull pilosa/pilosa:latest
+```
+
+Then, create a virtual network to attach our containers. We are going to name our network `pilosanet`:
+
+```
+docker network create pilosanet
+```
+
+Let's run the first Pilosa node and attach it to that virtual network. We set the first node as the cluster coordinator and use its address as the gossip seed. And also set the server address to `pilosa1`:
+```
+docker run -it --rm --name pilosa1 -p 10101:10101 --network=pilosanet pilosa/pilosa:latest server --bind pilosa1 --cluster.coordinator=true --gossip.seeds=pilosa1:14000
+```
+
+Let's run the second Pilosa node and attach it to the virtual network as well. Note that we set the address of the gossip seed to the address of the first node:
+```
+docker run -it --rm --name pilosa2 --network=pilosanet pilosa/pilosa:latest server --bind pilosa2 --gossip.seeds=pilosa1:14000
+```
+
+Let's test that the nodes in the cluster connected with each other:
+``` request
+curl localhost:10101/status
+```
+``` response
+{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}]}
+```
+
+And similarly for the second node:
+``` request
+curl localhost:10102/status
+```
+``` response
+{"state":"NORMAL","nodes":[{"id":"2e8332d0-1fee-44dd-a359-e0d6ecbcefc1","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"8c0dbcdc-9503-4265-8ad2-ba85a4bb10fa","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}]}
+```
+The corresponding [Docker Compose](https://docs.docker.com/compose/) file is below:
+
+```yaml
+version: '2'
+services: 
+  pilosa1:
+    image: pilosa/pilosa:latest
+    ports:
+      - "10101:10101"
+    environment:
+      - PILOSA_CLUSTER_COORDINATOR=true
+      - PILOSA_GOSSIP_SEEDS=pilosa1:14000
+    networks:
+      - pilosanet
+    entrypoint:
+      - /pilosa
+      - server
+      - --bind
+      - "pilosa1:10101"
+  pilosa2:
+    image: pilosa/pilosa:latest
+    environment:
+      - PILOSA_GOSSIP_SEEDS=pilosa1:14000
+    networks:
+      - pilosanet
+    entrypoint:
+      - /pilosa
+      - server
+      - --bind
+      - "pilosa2:10101"
+networks: 
+  pilosanet:
+```
+
+#### Running a Docker Swarm
+
+It is very easy to run a Pilosa Cluster on different servers using [Docker Swarm mode](https://docs.docker.com/engine/swarm/). All we have to do is creating an overlay network instead of the bridge network.
+
+The instructions in this section require Docker 17.06 and better. Although it is possible to run a Docker swarm on MacOS or Windows, it is easiest to run it on Linux. So we assume you are trying these instructions on Linux, probably on the cloud.
+
+We are going to use two servers: the manager node runs in the first server and a worker node in the second server.
+
+Docker nodes require some ports to be accesible from outside. Before carrying on, make sure the following ports are open on all nodes: TCP/2377, TCP/7946, UDP/7946, UDP/4789.
+
+Let's initialize the swarm first. Run the following on the manager:
+```
+docker swarm init --advertise-addr=IP-ADDRESS
+```
+
+Virtual machines running on the cloud usually have at least two network interfaces: the external interface and the internal interface. Use the IP of the external interface.
+
+The output of the command above should be similar to:
+```
+To add a manager to this swarm, run the following command:
+
+    docker swarm join --token SOME-TOKEN MANAGER-IP-ADDRESS:2377
+```
+
+Let's make the worker node join the manager. Copy/paste the command above in a shell on the worker, replacing the token and IP address with the correct values. You may neeed to add `--advertise-addr=WORKER-EXTERNAL-IP-ADDRESS` parameter if the worker has more than one network interface:
+```
+docker swarm join --token SOME-TOKEN MANAGER-IP-ADDRESS:2377
+```
+
+Run the following on the manager to check that the worker joined to the swarm:
+```
+docker node ls
+```
+
+Which should output:
+
+ID|HOSTNAME|STATUS|AVAILABILITY|MANAGER STATUS|ENGINE VERSION
+---|--------|------|------------|--------------|-------------
+MANAGER-ID *|swarm1|Ready|Active|Leader|18.05.0-ce|
+WORKER-ID|swarm2|Ready|Active||18.05.0-ce|
+
+If you have created the `pilosanet` network before, delete it before carrying on, otherwise skip to the next step:
+```
+docker network rm pilosanet
+```
+
+Let's create the `pilosanet` network, but with `overlay` type this time. We should also make this network attachable in order to be able to attach containers to it. Run the following on the manager:
+```
+docker network create -d overlay pilosanet --attachable
+```
+
+We can now create the Pilosa containers. Let's start the coordinator node first. Run the following on one of the servers:
+```
+docker run -it --rm --name pilosa1 --network=pilosanet pilosa/pilosa:latest server --bind pilosa1 --cluster.coordinator=true --gossip.seeds=pilosa1:14000
+```
+
+And the following on the other server:
+```
+docker run -it --rm --name pilosa2 --network=pilosanet pilosa/pilosa:latest server --bind pilosa2 --gossip.seeds=pilosa1:14000
+```
+
+These were the same commands we used in the previous section except the port mapping! Let's run another container on the same virtual network to read the status from the coordinator:
+``` request
+docker run -it --rm --network=pilosanet --name shell alpine wget -q -O- pilosa1:10101/status
+```
+``` response
+{"state":"NORMAL","nodes":[{"id":"3e3b0abd-1945-441a-a01f-5a28272972f5","uri":{"scheme":"http","host":"pilosa1","port":10101},"isCoordinator":true},{"id":"71ed27cc-9443-4f41-88fb-1c22f92bf695","uri":{"scheme":"http","host":"pilosa2","port":10101},"isCoordinator":false}]}
+```
+
+You can add as many as worker nodes to both the swarm and the Pilosa cluster using the steps above.
+
+#### What's Next?
+
+Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administration/) to learn more about making the most of your Pilosa cluster and [Configuration Documentation](https://www.pilosa.com/docs/latest/configuration/) to see the available options to configure Pilosa.
+
+Refer to the [Docker documentation](https://docs.docker.com) to see your options about running Docker containers. The [Networking with overlay networks](https://docs.docker.com/network/network-tutorial-overlay/) is a detailed overview of the Docket swarm mode and overlay networks.
+
 
 ### Using Integer Field Values
 

handler.go

@@ -437,7 +437,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
 	}
 
 	_, err = h.API.CreateIndex(r.Context(), indexName, req.Options)
-	if err == ErrIndexExists {
+	if errors.Cause(err) == ErrIndexExists {
 		http.Error(w, err.Error(), http.StatusConflict)
 		return
 	} else if err != nil {
@@ -464,7 +464,7 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request
 
 	attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks)
 	if err != nil {
-		if err == ErrIndexNotFound {
+		if errors.Cause(err) == ErrIndexNotFound {
 			http.Error(w, err.Error(), http.StatusNotFound)
 		} else {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -505,7 +505,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
 	}
 	_, err = h.API.CreateFrame(r.Context(), indexName, frameName, req.Options)
 	if err != nil {
-		switch err {
+		switch errors.Cause(err) {
 		case ErrIndexNotFound:
 			http.Error(w, err.Error(), http.StatusNotFound)
 		case ErrFrameExists:
@@ -573,7 +573,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
 
 	err := h.API.DeleteFrame(r.Context(), indexName, frameName)
 	if err != nil {
-		if err == ErrIndexNotFound {
+		if errors.Cause(err) == ErrIndexNotFound {
 			if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil {
 				h.Logger.Printf("response encoding error: %s", err)
 			}
@@ -612,7 +612,7 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := h.API.CreateField(r.Context(), indexName, frameName, field); err != nil {
-		if err == ErrFrameNotFound {
+		if errors.Cause(err) == ErrFrameNotFound {
 			http.Error(w, err.Error(), http.StatusNotFound)
 		} else {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -641,7 +641,7 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request)
 	fieldName := mux.Vars(r)["field"]
 
 	if err := h.API.DeleteField(r.Context(), indexName, frameName, fieldName); err != nil {
-		if err == ErrFrameNotFound {
+		if errors.Cause(err) == ErrFrameNotFound {
 			http.Error(w, err.Error(), http.StatusNotFound)
 		} else {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -661,7 +661,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) {
 
 	fields, err := h.API.Fields(r.Context(), indexName, frameName)
 	if err != nil {
-		switch err {
+		switch errors.Cause(err) {
 		case ErrIndexNotFound:
 			fallthrough
 		case ErrFrameNotFound:
@@ -691,7 +691,7 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) {
 
 	views, err := h.API.Views(r.Context(), indexName, frameName)
 	if err != nil {
-		if err == ErrFrameNotFound {
+		if errors.Cause(err) == ErrFrameNotFound {
 			http.Error(w, err.Error(), http.StatusNotFound)
 		} else {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -717,7 +717,7 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) {
 	viewName := mux.Vars(r)["view"]
 
 	if err := h.API.DeleteView(r.Context(), indexName, frameName, viewName); err != nil {
-		if err == ErrFrameNotFound {
+		if errors.Cause(err) == ErrFrameNotFound {
 			http.Error(w, err.Error(), http.StatusNotFound)
 		} else {
 			http.Error(w, err.Error(), http.StatusBadRequest)
@@ -751,7 +751,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request
 
 	attrs, err := h.API.FrameAttrDiff(r.Context(), indexName, frameName, req.Blocks)
 	if err != nil {
-		switch err {
+		switch errors.Cause(err) {
 		case ErrFragmentNotFound:
 			http.Error(w, err.Error(), http.StatusNotFound)
 		default:
@@ -878,7 +878,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := h.API.Import(r.Context(), req); err != nil {
-		switch err {
+		switch errors.Cause(err) {
 		case ErrIndexNotFound:
 			fallthrough
 		case ErrFrameNotFound:
@@ -931,7 +931,7 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request)
 	}
 
 	if err = h.API.ImportValue(r.Context(), req); err != nil {
-		switch err {
+		switch errors.Cause(err) {
 		case ErrIndexNotFound:
 			fallthrough
 		case ErrFrameNotFound:
@@ -980,7 +980,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil {
-		switch err {
+		switch errors.Cause(err) {
 		case ErrFragmentNotFound:
 			break
 		case ErrClusterDoesNotOwnSlice:
@@ -1051,7 +1051,7 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request)
 	}
 
 	if err = h.API.UnmarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil {
-		if err == ErrFrameNotFound {
+		if errors.Cause(err) == ErrFrameNotFound {
 			http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
 		} else {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -1065,7 +1065,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ
 	if err != nil {
 		if _, ok := err.(BadRequestError); ok {
 			http.Error(w, err.Error(), http.StatusBadRequest)
-		} else if err == ErrFragmentNotFound {
+		} else if errors.Cause(err) == ErrFragmentNotFound {
 			http.Error(w, err.Error(), http.StatusNotFound)
 		} else {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -1091,7 +1091,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
 
 	blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice)
 	if err != nil {
-		if err == ErrFragmentNotFound {
+		if errors.Cause(err) == ErrFragmentNotFound {
 			http.Error(w, err.Error(), http.StatusNotFound)
 		} else {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -1131,7 +1131,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
 	}
 
 	err = h.API.RestoreFrame(r.Context(), indexName, frameName, host)
-	switch err {
+	switch errors.Cause(err) {
 	case nil:
 		break
 	case ErrFrameNotFound:

server.go

@@ -307,8 +307,13 @@ func (s *Server) Open() error {
 
 	// Serve HTTP.
 	go func() {
-		err := http.Serve(s.ln, s.handler)
-		if err != nil {
+		server := &http.Server{Handler: s.handler}
+		go func() {
+			<-s.closing
+			server.Close()
+		}()
+		err := server.Serve(s.ln)
+		if err != nil && err.Error() != "http: Server closed" {
 			s.logger.Printf("HTTP handler terminated with error: %s\n", err)
 		}
 	}()

server/cluster_test.go

@@ -187,11 +187,9 @@ func TestClusterResize_AddNode(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		time.Sleep(1 * time.Second)
-
-		if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
-		} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		} else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
 		}
 	})
@@ -231,12 +229,9 @@ func TestClusterResize_AddNode(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Give the cluster time to settle.
-		time.Sleep(1 * time.Second)
-
-		if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
-		} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		} else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
 		}
 	})
@@ -286,12 +281,9 @@ func TestClusterResize_AddNode(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Give the cluster time to settle.
-		time.Sleep(1 * time.Second)
-
-		if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
-		} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		} else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
 		}
 	})
@@ -341,12 +333,9 @@ func TestClusterResize_AddNode(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Give the cluster time to settle.
-		time.Sleep(1 * time.Second)
-
-		if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
-		} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		} else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
 		}
 	})
@@ -395,14 +384,11 @@ func TestCluster_GossipMembership(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Give the cluster time to settle.
-		time.Sleep(1 * time.Second)
-
-		if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		if !checkClusterState(m0.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
-		} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		} else if !checkClusterState(m1.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
-		} else if m2.Server.Cluster.State() != pilosa.ClusterStateNormal {
+		} else if !checkClusterState(m2.Server.Cluster, pilosa.ClusterStateNormal, 1000) {
 			t.Fatalf("unexpected node2 cluster state: %s", m2.Server.Cluster.State())
 		}
 
@@ -497,3 +483,15 @@ func TestClusterResize_RemoveNode(t *testing.T) {
 		}
 	})
 }
+
+// checkClusterState polls a given cluster for its state until it
+// receives a matching state. It polls up to n times before returning.
+func checkClusterState(c *pilosa.Cluster, state string, n int) bool {
+	for i := 0; i < n; i++ {
+		if c.State() == state {
+			return true
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	return false
+}