Merged with master

2026-08-28 10:54:59 +00:00 · 2018-11-15 21:19:58 +03:00 · 2018-11-15 21:19:58 +03:00 · 0fe5d56f68
commit 0fe5d56f68
parent 2bab6eb5ec faac64dc99
19 changed files with 416 additions and 151 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -44,6 +44,12 @@ jobs:
    <<: *base-test
    environment:
      GOARCH: 386
+  cluster-tests:
+    <<: *defaults
+    steps:
+      - *fast-checkout
+      - setup_remote_docker
+      - run: make clustertests-build
  prerelease:
    <<: *base-test
    steps:
@ -99,6 +105,9 @@ workflows:
      - test-golang-1.10-386:
          requires:
            - build
+      - cluster-tests:
+          requires:
+            - build
      - prerelease:
          requires:
            - linter
--- a/.travis.yml
+++ b/.travis.yml
@ -1,52 +0,0 @@
-language: go
-go:
-  - "1.10" # Use string, as 1.10==1.1 if interpreted as float.
-  - master
-env:
-  global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
-    - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA="
-    - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
-  matrix:
-    - GOARCH=386
-    - GOARCH=386 ENTERPRISE=1
-    - GOARCH=amd64
-    - GOARCH=amd64 ENTERPRISE=1
-cache:
-  directories:
-    vendor
-install:
-  - make -B install-dep vendor
-script: make test
-jobs:
-  include:
-    - stage: metalinter
-      install:
-        - make -B install-dep vendor install-gometalinter
-      script: make gometalinter
-      env:
-        - GOARCH=amd64
-    - stage: deploy
-      script: skip
-      go: "1.10"
-      env:
-        - GOARCH=amd64
-      deploy:
-        - provider: script
-          script: pip install awscli --user `whoami` && make -B prerelease
-          skip_cleanup: true
-          on:
-            all_branches: true
-            repo: pilosa/pilosa
-            tags: false
-stages:
-  - metalinter
-  - test
-  - deploy
-matrix:
-  # Excluding or allowing failures on non-primary matrix configurations due to long running times.
-  fast_finish: true
-  allow_failures:
-    - go: master
-notifications:
-  slack:
-    secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -22,7 +22,7 @@ If you want to help but you aren't sure where to start, check out our [github la

 ### Development Environment

- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../master/.travis.yml) for the most up-to-date information.
+- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information.

 - Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH).

--- a/26
+++ b/26
@ -0,0 +1,26 @@
+# This Dockerfile is used for cluster testing - it produces a much larger image
+# and includes all of Go as well as some utilities.
+
+FROM golang:1.11
+
+LABEL maintainer "dev@pilosa.com"
+
+COPY . /go/src/github.com/pilosa/pilosa/
+
+RUN cd /go/src/github.com/pilosa/pilosa \
+    && CGO_ENABLED=0 make install-dep install FLAGS="-a"
+
+# download pumba for fault injection
+ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
+RUN chmod +x /pumba
+
+RUN cp /go/bin/pilosa /pilosa
+
+COPY LICENSE /LICENSE
+COPY NOTICE /NOTICE
+
+EXPOSE 10101
+VOLUME /data
+
+ENTRYPOINT ["bash", "-c"]
+CMD ["/pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
--- a/Gopkg.lock
+++ b/Gopkg.lock
@ -189,6 +189,15 @@
  revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
  version = "v1.2.0"

+[[projects]]
+  name = "github.com/pilosa/go-pilosa"
+  packages = [
+    ".",
+    "gopilosa_pbuf"
+  ]
+  revision = "4e7807f5ad779407936744057cd17332046b6c3c"
+  version = "v1.1.0"
+
 [[projects]]
  name = "github.com/pkg/errors"
  packages = ["."]
@ -324,6 +333,6 @@
 [solve-meta]
  analyzer-name = "dep"
  analyzer-version = 1
-  inputs-digest = "8290156ce8b4066c46ab83d743f4c81df0a17e148415bb1ee8409a51ac4c3ba4"
+  inputs-digest = "be318fa4f2a72e7e849b2faff1ce9300deaaf2d24cf76100f4b538abed86295b"
  solver-name = "gps-cdcl"
  solver-version = 1
--- a/15
+++ b/15
@ -68,6 +68,21 @@ release: check-clean
 	$(MAKE) release-build GOOS=linux GOARCH=386
 	$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1

+# Run cluster integration tests using docker. Requires docker daemon to be
+# running. This will catch changes to internal/clustertests/*.go, but if you
+# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
+# pilosa image.
+clustertests:
+	docker-compose -f internal/clustertests/docker-compose.yml down
+	docker-compose -f internal/clustertests/docker-compose.yml build client1
+	docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1
+
+
+# Like clustertests, but rebuilds all images.
+clustertests-build:
+	docker-compose -f internal/clustertests/docker-compose.yml down
+	docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 --build
+
 # Create prerelease builds
 prerelease: vendor
 	$(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\)
--- a/README.md
+++ b/README.md
@ -4,7 +4,7 @@
    </a>
 </p>

-[![Build Status](https://travis-ci.org/pilosa/pilosa.svg?branch=master)](https://travis-ci.org/pilosa/pilosa)
+[![CircleCI](https://circleci.com/gh/pilosa/pilosa/tree/master.svg?style=shield)](https://circleci.com/gh/pilosa/pilosa/tree/master)
 [![GoDoc](https://godoc.org/github.com/pilosa/pilosa?status.svg)](https://godoc.org/github.com/pilosa/pilosa)
 [![Go Report Card](https://goreportcard.com/badge/github.com/pilosa/pilosa)](https://goreportcard.com/report/github.com/pilosa/pilosa)
 [![license](https://img.shields.io/github/license/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/blob/master/LICENSE)
--- a/apimethod_string.go
+++ b/apimethod_string.go
@ -4,9 +4,9 @@ package pilosa

 import "strconv"

-const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews"
+const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFieldapiFieldAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSetCoordinatorapiShardNodesapiViews"

-var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 73, 86, 98, 118, 135, 143, 159, 168, 182, 190, 206, 214, 234, 247, 261, 278, 291, 299}
+var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 166, 182, 191, 205, 213, 229, 237, 257, 270, 284, 301, 314, 322}

 func (i apiMethod) String() string {
 	if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) {
--- a/cluster.go
+++ b/cluster.go
@ -65,10 +65,11 @@ type Node struct {
 	ID            string `json:"id"`
 	URI           URI    `json:"uri"`
 	IsCoordinator bool   `json:"isCoordinator"`
+	State         string `json:"state"`
 }

 func (n Node) String() string {
-	return fmt.Sprintf("Node: %s", n.ID)
+	return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID[:6])
 }

 // Nodes represents a list of nodes.
@ -456,7 +457,19 @@ func (c *cluster) unprotectedSetState(state string) {
 	}
 }

+func (c *cluster) setMyNodeState(state string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.Node.State = state
+	for i, n := range c.nodes {
+		if n.ID == c.Node.ID {
+			c.nodes[i].State = state
+		}
+	}
+}
+
 func (c *cluster) setNodeState(state string) error { // nolint: unparam
+	c.setMyNodeState(state)
 	if c.isCoordinator() {
 		return c.receiveNodeState(c.Node.ID, state)
 	}
@ -486,11 +499,23 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error {
 	}

 	c.Topology.mu.Lock()
-	c.Topology.nodeStates[nodeID] = state
+	changed := false
+	if c.Topology.nodeStates[nodeID] != state {
+		changed = true
+		c.Topology.nodeStates[nodeID] = state
+		for i, n := range c.nodes {
+			if n.ID == nodeID {
+				c.nodes[i].State = state
+			}
+		}
+	}
 	c.Topology.mu.Unlock()
 	c.logger.Printf("received state %s (%s)", state, nodeID)

-	return c.unprotectedSetStateAndBroadcast(c.determineClusterState())
+	if changed {
+		return c.unprotectedSetStateAndBroadcast(c.determineClusterState())
+	}
+	return nil
 }

 // determineClusterState is unprotected.
@ -932,7 +957,6 @@ func (c *cluster) waitForStarted() error {
 		<-c.joining
 		c.logger.Printf("joining has completed")
 	}
-
 	return nil
 }

@ -1043,8 +1067,9 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error {
 		return nil
 	}
 	// Broadcast cluster status changes to the cluster.
-	c.logger.Printf("broadcasting ClusterStatus: %s", state)
-	return c.broadcaster.SendSync(c.unprotectedStatus()) // TODO fix c.Status
+	status := c.unprotectedStatus()
+	c.logger.Printf("broadcasting ClusterStatus: %s", status)
+	return c.broadcaster.SendSync(status) // TODO fix c.Status

 }

@ -1625,7 +1650,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {

 	switch e.Event {
 	case NodeJoin:
-		c.logger.Printf("received NodeJoin event: %v", e)
+		c.logger.Printf("nodeJoin of %s on %s", e.Node.URI, c.Node.URI)
 		// Ignore the event if this is not the coordinator.
 		if !c.isCoordinator() {
 			return nil
@ -1660,6 +1685,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
 func (c *cluster) nodeJoin(node *Node) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	c.logger.Printf("NodeJoin event on coordinator, node: %s, id: %s", node.URI, node.ID)
 	if c.needTopologyAgreement() {
 		// A host that is not part of the topology can't be added to the STARTING cluster.
 		if !c.Topology.ContainsID(node.ID) {
@ -1688,11 +1714,10 @@ func (c *cluster) nodeJoin(node *Node) error {

 		if c.haveTopologyAgreement() && c.allNodesReady() {
 			return c.unprotectedSetStateAndBroadcast(ClusterStateNormal)
-		} else {
-			// Send the status to the remote node. This lets the remote node
-			// know that it can proceed with opening its Holder.
-			return c.sendTo(node, c.unprotectedStatus())
 		}
+		// Send the status to the remote node. This lets the remote node
+		// know that it can proceed with opening its Holder.
+		return c.sendTo(node, c.unprotectedStatus())
 	}

 	// If the cluster already contains the node, just send it the cluster status.
@ -1796,6 +1821,10 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {

 	// Add all nodes from the coordinator.
 	for _, node := range officialNodes {
+		if node.ID == c.Node.ID && node.State != c.Node.State {
+			c.logger.Printf("mismatched state in mergeClusterStatus got %v have %v", node.State, c.Node.State)
+			go c.setNodeState(c.Node.State)
+		}
 		if err := c.addNode(node); err != nil {
 			return errors.Wrap(err, "adding node")
 		}
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@ -185,8 +185,8 @@ func TestFragSources(t *testing.T) {
 				"node0": {},
 				"node1": {},
 				"node2": {
-					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
+					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
+					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@ -197,11 +197,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
+					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)},
 				},
 				"node1": {
-					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
+					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
+					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@ -212,11 +212,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
+					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
+					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
 				},
 				"node1": {
-					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
+					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)},
 				},
 				"node2": {},
 			},
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@ -534,6 +534,7 @@ func encodeNode(n *pilosa.Node) *internal.Node {
 		ID:            n.ID,
 		URI:           encodeURI(n.URI),
 		IsCoordinator: n.IsCoordinator,
+		State:         n.State,
 	}
 }

@ -789,6 +790,7 @@ func decodeNode(node *internal.Node, m *pilosa.Node) {
 	m.ID = node.ID
 	decodeURI(node.URI, &m.URI)
 	m.IsCoordinator = node.IsCoordinator
+	m.State = node.State
 }

 func decodeURI(i *internal.URI, m *pilosa.URI) {
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@ -18,6 +18,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net"
@ -49,6 +50,7 @@ type memberSet struct {
 	Logger pilosa.Logger

 	logger    *log.Logger
+	logOutput io.Writer
 	transport *Transport

 	eventReceiver *eventReceiver
@ -156,6 +158,13 @@ func WithLogger(logger *log.Logger) memberSetOption {
 	}
 }

+func WithLogOutput(o io.Writer) memberSetOption {
+	return func(g *memberSet) error {
+		g.logOutput = o
+		return nil
+	}
+}
+
 // NewMemberSet returns a new instance of GossipMemberSet based on options.
 func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*memberSet, error) {
 	host := api.Node().URI.Host
@ -220,7 +229,11 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem
 	conf.Delegate = g
 	conf.SecretKey = gossipKey
 	conf.Events = ger
-	conf.Logger = g.logger
+	if g.logOutput != nil {
+		conf.LogOutput = g.logOutput
+	} else {
+		conf.Logger = g.logger
+	}

 	g.config = &config{
 		memberlistConfig: conf,
--- a/internal/clustertests/Dockerfile
+++ b/internal/clustertests/Dockerfile
@ -0,0 +1,3 @@
+FROM ptest
+
+COPY . /go/src/github.com/pilosa/pilosa/internal/clustertests
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@ -0,0 +1,81 @@
+package clustertest
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/pilosa/pilosa"
+	picli "github.com/pilosa/pilosa/http"
+)
+
+func TestClusterStuff(t *testing.T) {
+	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
+		t.Skip()
+	}
+	cli, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil))
+	if err != nil {
+		t.Fatalf("getting client: %v", err)
+	}
+
+	t.Run("long pause", func(t *testing.T) {
+		err := cli.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{})
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		err = cli.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100})
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
+
+		data := make([]pilosa.Bit, 10)
+		for i := 0; i < 1000; i++ {
+			data[i%10].RowID = 0
+			data[i%10].ColumnID = uint64((i/10)*pilosa.ShardWidth + i%10)
+			shard := uint64(i / 10)
+			if i%10 == 9 {
+				err = cli.Import(context.Background(), "testidx", "testf", shard, data)
+				if err != nil {
+					t.Fatalf("importing: %v", err)
+				}
+			}
+		}
+
+		r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
+		if err != nil {
+			t.Fatalf("count querying: %v", err)
+		}
+		if r.Results[0].(uint64) != 1000 {
+			t.Fatalf("count after import is %d", r.Results[0].(uint64))
+		}
+
+		pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s")
+		pcmd.Stdout = os.Stdout
+		pcmd.Stderr = os.Stderr
+		t.Log("pausing pilosa3 for 10s")
+		err = pcmd.Start()
+		if err != nil {
+			t.Fatalf("starting pumba command: %v", err)
+		}
+		err = pcmd.Wait()
+		if err != nil {
+			t.Fatalf("waiting on pumba pause cmd: %v", err)
+		}
+
+		// TODO change the sleep to wait for status to return to NORMAL - need support in internal client for getting status
+		t.Log("done with pause, waiting for stability")
+		time.Sleep(time.Second * 3)
+		t.Log("done waiting for stability")
+
+		r, err = cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
+		if err != nil {
+			t.Fatalf("count querying: %v", err)
+		}
+		if r.Results[0].(uint64) != 1000 {
+			t.Fatalf("count after import is %d", r.Results[0].(uint64))
+		}
+	})
+
+}
--- a/internal/clustertests/docker-compose.yml
+++ b/internal/clustertests/docker-compose.yml
@ -0,0 +1,55 @@
+version: '2'
+services: 
+  pilosa1:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-clustertests
+    image: ptest
+    ports:
+      - "33455:10101"
+    environment:
+      - PILOSA_CLUSTER_COORDINATOR=true
+      - PILOSA_GOSSIP_SEEDS=pilosa1:14000
+    networks:
+      - pilosanet
+    command:
+      - "/pilosa server --bind pilosa1:10101"
+  pilosa2:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-clustertests
+    image: ptest
+    ports:
+      - "33456:10101"
+    environment:
+      - PILOSA_GOSSIP_SEEDS=pilosa1:14000
+    networks:
+      - pilosanet
+    command:
+      - "/pilosa server --bind pilosa2:10101"
+  pilosa3:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-clustertests
+    image: ptest
+    ports:
+      - "33457:10101"
+    environment:
+      - PILOSA_GOSSIP_SEEDS=pilosa1:14000,pilosa2:14000
+    networks:
+      - pilosanet
+    command:
+      - "/pilosa server --bind pilosa3:10101"
+  client1:
+    build:
+      context: .
+    environment:
+      - ENABLE_PILOSA_CLUSTER_TESTS=1
+    networks:
+      - pilosanet
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command:
+      - "go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests"
+networks: 
+  pilosanet:
--- a/internal/private.pb.go
+++ b/internal/private.pb.go
@ -541,6 +541,7 @@ type Node struct {
 	ID            string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"`
 	URI           *URI   `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"`
 	IsCoordinator bool   `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"`
+	State         string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"`
 }

 func (m *Node) Reset()                    { *m = Node{} }
@ -569,6 +570,13 @@ func (m *Node) GetIsCoordinator() bool {
 	return false
 }

+func (m *Node) GetState() string {
+	if m != nil {
+		return m.State
+	}
+	return ""
+}
+
 type NodeStateMessage struct {
 	NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"`
 	State  string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
@ -1750,6 +1758,12 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i++
 	}
+	if len(m.State) > 0 {
+		dAtA[i] = 0x22
+		i++
+		i = encodeVarintPrivate(dAtA, i, uint64(len(m.State)))
+		i += copy(dAtA[i:], m.State)
+	}
 	return i, nil
 }

@ -2694,6 +2708,10 @@ func (m *Node) Size() (n int) {
 	if m.IsCoordinator {
 		n += 2
 	}
+	l = len(m.State)
+	if l > 0 {
+		n += 1 + l + sovPrivate(uint64(l))
+	}
 	return n
 }

@ -5245,6 +5263,35 @@ func (m *Node) Unmarshal(dAtA []byte) error {
 				}
 			}
 			m.IsCoordinator = bool(v != 0)
+		case 4:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field State", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPrivate
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.State = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7417,75 +7464,76 @@ var (
 func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }

 var fileDescriptorPrivate = []byte{
-	// 1113 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45,
-	0x18, 0x66, 0x0f, 0x76, 0xec, 0xdf, 0x75, 0x9a, 0x6c, 0x69, 0xd9, 0x02, 0x0a, 0x61, 0x54, 0xd1,
-	0x50, 0x89, 0x50, 0xb5, 0x37, 0x9c, 0x2a, 0x95, 0xc4, 0xa1, 0x2c, 0x25, 0xa5, 0xcc, 0xa6, 0xb9,
-	0xeb, 0xc5, 0xc4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x12, 0xf7, 0x82, 0x5b, 0x90,
-	0x78, 0x01, 0xc4, 0x93, 0xf0, 0x08, 0x5c, 0xf2, 0x08, 0x28, 0xbc, 0x08, 0x9a, 0x7f, 0x66, 0x76,
-	0x37, 0x8e, 0x43, 0xa2, 0xc0, 0xdd, 0xfc, 0xdf, 0x7f, 0x3e, 0xae, 0x0d, 0xfd, 0x49, 0x9e, 0x1c,
-	0x32, 0xc9, 0xd7, 0x27, 0xb9, 0x90, 0x22, 0xe8, 0x24, 0x99, 0xe4, 0x79, 0xc6, 0x52, 0xf2, 0x04,
-	0xba, 0x51, 0x36, 0xe2, 0xc7, 0xdb, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x29, 0x9f, 0x16, 0xa1, 0xb7,
-	0xea, 0xac, 0x75, 0x28, 0xbe, 0x83, 0x0f, 0x60, 0x71, 0x27, 0x67, 0xc3, 0x83, 0xad, 0xe3, 0xa4,
-	0x90, 0x3c, 0x1b, 0xf2, 0xd0, 0x47, 0xee, 0x0c, 0x4a, 0x7e, 0x77, 0xe0, 0xda, 0x57, 0x09, 0x4f,
-	0x47, 0xdf, 0x4d, 0x64, 0x22, 0xb2, 0x22, 0x78, 0x17, 0xba, 0x9b, 0x6c, 0xb8, 0xcf, 0x77, 0xa6,
-	0x13, 0x8e, 0x16, 0xbb, 0xb4, 0x06, 0x2a, 0x6e, 0x9c, 0xbc, 0xd6, 0x16, 0xfb, 0xb4, 0x06, 0x82,
-	0x55, 0xe8, 0xed, 0x24, 0x63, 0xfe, 0x7d, 0xc9, 0x32, 0x59, 0x8e, 0xc3, 0x16, 0x6a, 0x37, 0x21,
-	0x15, 0x2a, 0x1a, 0xee, 0x20, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0xdb, 0x49, 0x16, 0x76, 0x57, 0x9d,
-	0x35, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x38, 0x04, 0x83, 0xb0, 0xe3, 0x2a, 0xc5, 0x5e, 0x9d, 0x22,
-	0x21, 0xb0, 0x18, 0x8d, 0x27, 0x22, 0x97, 0x94, 0x17, 0x13, 0x91, 0x15, 0x68, 0x69, 0x2b, 0xcf,
-	0x43, 0x07, 0x8d, 0xab, 0x27, 0xf9, 0x11, 0x96, 0x36, 0x52, 0x31, 0x3c, 0x18, 0x30, 0xc9, 0x28,
-	0xff, 0xa1, 0xe4, 0x85, 0x0c, 0xde, 0x84, 0x16, 0xd6, 0xce, 0xc8, 0x69, 0x42, 0xa1, 0x58, 0x87,
-	0xd0, 0xd5, 0x28, 0x12, 0x0a, 0x45, 0x7d, 0xac, 0x84, 0x4f, 0x35, 0xa1, 0xd0, 0x78, 0x9f, 0xe5,
-	0x23, 0xac, 0x80, 0x4f, 0x35, 0xa1, 0x62, 0xdc, 0x4d, 0xf8, 0x91, 0x49, 0x1b, 0xdf, 0x24, 0x82,
-	0xe5, 0x86, 0x7f, 0x13, 0xe6, 0x2d, 0x68, 0x53, 0x71, 0x14, 0x0d, 0x8a, 0xd0, 0x59, 0xf5, 0xd6,
-	0x7c, 0x6a, 0x28, 0x2c, 0xae, 0x48, 0xcb, 0x71, 0xa6, 0x58, 0x2e, 0xb2, 0x6a, 0x80, 0xdc, 0x86,
-	0x16, 0x56, 0x5a, 0x65, 0x59, 0xeb, 0xaa, 0x27, 0xf9, 0xc9, 0x81, 0xee, 0x36, 0x3b, 0xc6, 0x30,
-	0x8a, 0xe0, 0x11, 0x74, 0x62, 0xc9, 0xb2, 0x91, 0x0a, 0x50, 0x09, 0xf5, 0x1e, 0xbc, 0xbf, 0x6e,
-	0x07, 0x67, 0xbd, 0x12, 0x5b, 0xb7, 0x32, 0x5b, 0x99, 0xcc, 0xa7, 0xb4, 0x52, 0x79, 0xfb, 0x73,
-	0xe8, 0x9f, 0x62, 0x29, 0x7f, 0x07, 0x7c, 0x6a, 0xab, 0x7a, 0xc0, 0xa7, 0x2a, 0xff, 0x43, 0x96,
-	0x96, 0x1c, 0x6b, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x5d, 0x08, 0x36, 0x73, 0xce,
-	0x24, 0x47, 0x27, 0xdb, 0xbc, 0x28, 0xd8, 0x2b, 0x7e, 0x7e, 0xc5, 0x75, 0x15, 0xdd, 0x66, 0x15,
-	0xab, 0x3e, 0x78, 0x8d, 0x3e, 0x90, 0x7b, 0x10, 0x0c, 0x78, 0xca, 0x25, 0x37, 0x53, 0xff, 0x2f,
-	0x76, 0x49, 0x6c, 0x63, 0xb8, 0x58, 0x36, 0xb8, 0x0b, 0xbe, 0x5a, 0x21, 0x0c, 0xa1, 0xf7, 0xe0,
-	0x46, 0x5d, 0xa7, 0x6a, 0xbb, 0x28, 0x0a, 0x90, 0xd4, 0x1a, 0xc5, 0x78, 0x2e, 0x4c, 0x6c, 0xce,
-	0x28, 0xdd, 0x33, 0xae, 0x3c, 0x74, 0x75, 0xab, 0x76, 0xd5, 0x5c, 0x3f, 0xe3, 0xed, 0xb1, 0x4d,
-	0xf7, 0xaa, 0xde, 0xc8, 0x10, 0xde, 0xd1, 0x16, 0xbe, 0x3c, 0x64, 0x49, 0xca, 0xf6, 0xd2, 0x4b,
-	0x76, 0x64, 0x4e, 0xe0, 0x21, 0x2c, 0xa0, 0x6e, 0x34, 0x30, 0x5b, 0x60, 0x49, 0xf2, 0xd2, 0xc8,
-	0xab, 0xd1, 0x7f, 0xc6, 0xc6, 0xdc, 0x58, 0xc3, 0x77, 0x95, 0xaf, 0x7b, 0x71, 0xbe, 0xca, 0xb1,
-	0x5a, 0x17, 0x75, 0xc2, 0x3c, 0xe5, 0x18, 0x09, 0xf2, 0x10, 0xda, 0xf1, 0x70, 0x9f, 0x8f, 0x59,
-	0xf0, 0x21, 0x2c, 0x60, 0x84, 0xbc, 0x30, 0x13, 0x7d, 0x7d, 0xa6, 0x53, 0xd4, 0xf2, 0xc9, 0xc0,
-	0x64, 0x36, 0x37, 0xa6, 0xbb, 0xd0, 0x46, 0xef, 0x45, 0xe8, 0xcf, 0x9a, 0x41, 0x9c, 0x1a, 0x36,
-	0xd9, 0x02, 0xef, 0x05, 0x8d, 0xd4, 0xa6, 0x62, 0x04, 0xd6, 0x8a, 0xa1, 0x94, 0xed, 0xaf, 0x45,
-	0x21, 0x4d, 0x9d, 0xf0, 0xad, 0xb0, 0xe7, 0x22, 0x97, 0x58, 0xa3, 0x3e, 0xc5, 0x37, 0x79, 0x09,
-	0xfe, 0x33, 0x31, 0xe2, 0xc1, 0x22, 0xb8, 0xd1, 0xc0, 0xd8, 0x70, 0xa3, 0x41, 0xf0, 0x1e, 0x9a,
-	0x37, 0xa5, 0xe9, 0xd7, 0x41, 0xbc, 0xa0, 0x11, 0x45, 0xc7, 0x77, 0xa0, 0x1f, 0x15, 0x9b, 0x42,
-	0xe4, 0xa3, 0x24, 0x63, 0x52, 0xe4, 0xe6, 0xb6, 0x9f, 0x06, 0xc9, 0x63, 0x58, 0x52, 0xe6, 0x63,
-	0xc9, 0x24, 0xb7, 0x9d, 0xbd, 0x05, 0x6d, 0x85, 0x55, 0xee, 0x0c, 0x85, 0xdb, 0xa6, 0xe4, 0x6c,
-	0x6f, 0x91, 0x20, 0xdf, 0x6a, 0x0b, 0x5b, 0x87, 0x3c, 0x93, 0x8d, 0xd9, 0x40, 0x1a, 0x0d, 0xf4,
-	0xa9, 0x26, 0x02, 0xa2, 0x53, 0x31, 0x31, 0x2f, 0xd6, 0x31, 0x2b, 0x94, 0x22, 0x8f, 0xfc, 0xe2,
-	0x00, 0xd8, 0x80, 0xca, 0xa2, 0x52, 0x71, 0xce, 0x57, 0x09, 0xd6, 0x6c, 0x8f, 0xcd, 0x5e, 0x2c,
-	0xd5, 0x52, 0x1a, 0xa7, 0x76, 0x06, 0x3e, 0xae, 0x67, 0x40, 0x37, 0xef, 0xe6, 0xcc, 0x0c, 0x68,
-	0xaf, 0xf5, 0x24, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0x3b, 0x0f, 0x1f, 0x55, 0xf3, 0xe0, 0xce, 0x9a,
-	0x44, 0xdc, 0x98, 0xb4, 0x53, 0xf1, 0x14, 0x7a, 0x0d, 0x78, 0xae, 0xc5, 0x35, 0xb8, 0x7e, 0x7a,
-	0xe3, 0xec, 0x25, 0x9f, 0x85, 0x49, 0x02, 0xfd, 0xcd, 0xb4, 0x2c, 0x24, 0xcf, 0x8d, 0x39, 0x75,
-	0xfe, 0x35, 0x50, 0x35, 0xaf, 0x06, 0xe6, 0xf7, 0x2f, 0xb8, 0x03, 0x2d, 0x55, 0x46, 0xbd, 0x38,
-	0x67, 0x6b, 0xac, 0x99, 0x64, 0x17, 0x3a, 0x1b, 0x71, 0xf4, 0x24, 0x17, 0xe5, 0x64, 0x6e, 0xd0,
-	0xf6, 0xab, 0xec, 0x9e, 0xfd, 0x2a, 0x7b, 0x67, 0xbe, 0xca, 0x7e, 0xf5, 0x55, 0x26, 0x31, 0x2c,
-	0xeb, 0xa3, 0xa8, 0xf6, 0xf5, 0x2a, 0xa7, 0xc5, 0x7e, 0x32, 0xbd, 0xc6, 0x27, 0x33, 0x86, 0x65,
-	0x7d, 0xb9, 0xfe, 0x4f, 0xa3, 0xbf, 0xb9, 0xb0, 0x4c, 0x79, 0x91, 0xbc, 0xe6, 0x51, 0x56, 0xc8,
-	0xbc, 0x1c, 0xaa, 0xeb, 0xa3, 0xf4, 0xbf, 0x11, 0x7b, 0xa6, 0xda, 0x1e, 0xd5, 0xc4, 0x65, 0x26,
-	0x3d, 0xb8, 0x0f, 0xbd, 0xd9, 0xed, 0x3c, 0x2b, 0xda, 0x14, 0x09, 0xee, 0xc3, 0x42, 0x2c, 0xca,
-	0x7c, 0x58, 0x8d, 0x6f, 0xe3, 0x22, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xc6, 0x6a, 0xb4, 0x2e,
-	0x58, 0x8d, 0x47, 0x33, 0xa3, 0x14, 0xb6, 0x51, 0xe1, 0xad, 0x5a, 0xe1, 0x14, 0x9b, 0x9e, 0x96,
-	0x26, 0x3f, 0x3b, 0x70, 0xad, 0x19, 0xc2, 0xa5, 0x16, 0xb7, 0xea, 0x88, 0x3b, 0xb7, 0x23, 0xde,
-	0xbc, 0x8e, 0xf8, 0x75, 0x47, 0xea, 0xaf, 0x7f, 0xab, 0xf1, 0xf5, 0x27, 0x07, 0x70, 0xfb, 0x4c,
-	0x9b, 0x36, 0xc5, 0x78, 0xa2, 0xe6, 0xe1, 0x3f, 0xb4, 0x4b, 0x9d, 0xb4, 0x3c, 0x37, 0x8d, 0xea,
-	0x52, 0x4d, 0x90, 0x4f, 0xe1, 0x66, 0xcc, 0x65, 0xa3, 0x49, 0x76, 0xda, 0x56, 0xc1, 0x7b, 0xc6,
-	0x8f, 0xce, 0x49, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xbe, 0x98, 0x8c, 0x98, 0xe4, 0x57, 0xd2, 0xde,
-	0x80, 0xce, 0x8e, 0x98, 0x88, 0x54, 0xbc, 0x9a, 0x5e, 0xb0, 0xf5, 0x21, 0x2c, 0xe8, 0xfb, 0xad,
-	0xcf, 0x48, 0x97, 0x5a, 0x92, 0xdc, 0x50, 0x03, 0x3d, 0x64, 0xe9, 0xb0, 0x4c, 0x55, 0x18, 0xea,
-	0x97, 0x61, 0xb1, 0xb1, 0xf4, 0xc7, 0xc9, 0x8a, 0xf3, 0xe7, 0xc9, 0x8a, 0xf3, 0xd7, 0xc9, 0x8a,
-	0xf3, 0xeb, 0xdf, 0x2b, 0x6f, 0xec, 0xb5, 0xf1, 0x9f, 0xc3, 0xc3, 0x7f, 0x02, 0x00, 0x00, 0xff,
-	0xff, 0xba, 0x1b, 0x62, 0x68, 0x4a, 0x0c, 0x00, 0x00,
+	// 1121 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0xc5,
+	0x17, 0xff, 0xef, 0x87, 0x1d, 0xfb, 0xb8, 0x4e, 0x93, 0xed, 0xbf, 0x61, 0x0b, 0x28, 0x84, 0x51,
+	0x45, 0x43, 0x25, 0x42, 0xd5, 0xde, 0xf0, 0x55, 0xa9, 0x24, 0x0e, 0x65, 0x29, 0x09, 0x65, 0x9c,
+	0xe4, 0x8e, 0x8b, 0x89, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0x98, 0xdd, 0xd9, 0x24, 0xee, 0x05, 0xb7,
+	0x20, 0xf1, 0x02, 0x88, 0x27, 0xe1, 0x11, 0xb8, 0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0x67, 0x66,
+	0x76, 0x37, 0x8e, 0x83, 0xa3, 0xc0, 0xdd, 0x9c, 0xdf, 0x99, 0xf9, 0x9d, 0xef, 0xb3, 0x36, 0x74,
+	0xc7, 0x59, 0x7c, 0xc2, 0x24, 0xdf, 0x18, 0x67, 0x42, 0x8a, 0xa0, 0x15, 0xa7, 0x92, 0x67, 0x29,
+	0x4b, 0xc8, 0x73, 0x68, 0x47, 0xe9, 0x90, 0x9f, 0xed, 0x70, 0xc9, 0x82, 0x00, 0xfc, 0x17, 0x7c,
+	0x92, 0x87, 0xde, 0x9a, 0xb3, 0xde, 0xa2, 0x78, 0x0e, 0xde, 0x83, 0xc5, 0xbd, 0x8c, 0x0d, 0x8e,
+	0xb7, 0xcf, 0xe2, 0x5c, 0xf2, 0x74, 0xc0, 0x43, 0x1f, 0xb5, 0x53, 0x28, 0xf9, 0xcd, 0x81, 0x5b,
+	0x5f, 0xc4, 0x3c, 0x19, 0x7e, 0x33, 0x96, 0xb1, 0x48, 0xf3, 0xe0, 0x6d, 0x68, 0x6f, 0xb1, 0xc1,
+	0x11, 0xdf, 0x9b, 0x8c, 0x39, 0x32, 0xb6, 0x69, 0x05, 0x94, 0xda, 0x7e, 0xfc, 0x5a, 0x33, 0x76,
+	0x69, 0x05, 0x04, 0x6b, 0xd0, 0xd9, 0x8b, 0x47, 0xfc, 0xdb, 0x82, 0xa5, 0xb2, 0x18, 0x85, 0x0d,
+	0x7c, 0x5d, 0x87, 0x94, 0xab, 0x48, 0xdc, 0x42, 0x15, 0x9e, 0x83, 0x25, 0xf0, 0x76, 0xe2, 0x34,
+	0x6c, 0xaf, 0x39, 0xeb, 0x1e, 0x55, 0x47, 0x44, 0xd8, 0x59, 0x08, 0x06, 0x61, 0x67, 0x65, 0x88,
+	0x9d, 0x2a, 0x44, 0x42, 0x60, 0x31, 0x1a, 0x8d, 0x45, 0x26, 0x29, 0xcf, 0xc7, 0x22, 0xcd, 0x91,
+	0x69, 0x3b, 0xcb, 0x42, 0x07, 0xc9, 0xd5, 0x91, 0xfc, 0x00, 0x4b, 0x9b, 0x89, 0x18, 0x1c, 0xf7,
+	0x98, 0x64, 0x94, 0x7f, 0x5f, 0xf0, 0x5c, 0x06, 0xff, 0x87, 0x06, 0xe6, 0xce, 0xdc, 0xd3, 0x82,
+	0x42, 0x31, 0x0f, 0xa1, 0xab, 0x51, 0x14, 0x14, 0x8a, 0xef, 0x31, 0x13, 0x3e, 0xd5, 0x82, 0x42,
+	0xfb, 0x47, 0x2c, 0x1b, 0x62, 0x06, 0x7c, 0xaa, 0x05, 0xe5, 0xe3, 0x41, 0xcc, 0x4f, 0x4d, 0xd8,
+	0x78, 0x26, 0x11, 0x2c, 0xd7, 0xec, 0x1b, 0x37, 0x57, 0xa0, 0x49, 0xc5, 0x69, 0xd4, 0xcb, 0x43,
+	0x67, 0xcd, 0x5b, 0xf7, 0xa9, 0x91, 0x30, 0xb9, 0x22, 0x29, 0x46, 0xa9, 0x52, 0xb9, 0xa8, 0xaa,
+	0x00, 0x72, 0x0f, 0x1a, 0x98, 0x69, 0x15, 0x65, 0xf5, 0x56, 0x1d, 0xc9, 0x8f, 0x0e, 0xb4, 0x77,
+	0xd8, 0x19, 0xba, 0x91, 0x07, 0x4f, 0xa1, 0xd5, 0x97, 0x2c, 0x1d, 0x2a, 0x07, 0xd5, 0xa5, 0xce,
+	0xe3, 0x77, 0x37, 0x6c, 0xe3, 0x6c, 0x94, 0xd7, 0x36, 0xec, 0x9d, 0xed, 0x54, 0x66, 0x13, 0x5a,
+	0x3e, 0x79, 0xf3, 0x53, 0xe8, 0x5e, 0x50, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27,
+	0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c, 0x73, 0xe5, 0x53, 0x2d, 0x7c, 0xe2, 0x7e, 0xe4, 0x90, 0x03,
+	0x08, 0xb6, 0x32, 0xce, 0x24, 0x47, 0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x75, 0xc6, 0x75,
+	0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a, 0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25,
+	0x37, 0x5d, 0xff, 0x0f, 0xbc, 0xa4, 0x6f, 0x7d, 0x98, 0x7f, 0x37, 0x78, 0x00, 0xbe, 0x1a, 0x21,
+	0x74, 0xa1, 0xf3, 0xf8, 0x4e, 0x95, 0xa7, 0x72, 0xba, 0x28, 0x5e, 0x20, 0x89, 0x25, 0x45, 0x7f,
+	0xe6, 0x06, 0x36, 0xa3, 0x95, 0x1e, 0x1a, 0x53, 0x1e, 0x9a, 0x5a, 0xa9, 0x4c, 0xd5, 0xc7, 0xcf,
+	0x58, 0x7b, 0x66, 0xc3, 0xbd, 0xa9, 0x35, 0x32, 0x80, 0xb7, 0x34, 0xc3, 0xe7, 0x27, 0x2c, 0x4e,
+	0xd8, 0x61, 0x72, 0xcd, 0x8a, 0xcc, 0x70, 0x3c, 0x84, 0x05, 0x7c, 0x1b, 0xf5, 0xcc, 0x14, 0x58,
+	0x91, 0x7c, 0x67, 0xee, 0xab, 0xd6, 0xdf, 0x65, 0x23, 0x6e, 0xd8, 0xf0, 0x5c, 0xc6, 0xeb, 0xce,
+	0x8f, 0x57, 0x19, 0x56, 0xe3, 0xa2, 0x56, 0x98, 0xa7, 0x0c, 0xa3, 0x40, 0x9e, 0x40, 0xb3, 0x3f,
+	0x38, 0xe2, 0x23, 0x16, 0xbc, 0x0f, 0x0b, 0xe8, 0x21, 0xcf, 0x4d, 0x47, 0xdf, 0x9e, 0xaa, 0x14,
+	0xb5, 0x7a, 0xd2, 0x33, 0x91, 0xcd, 0xf4, 0xe9, 0x01, 0x34, 0xd1, 0x7a, 0x1e, 0xfa, 0xd3, 0x34,
+	0x88, 0x53, 0xa3, 0x26, 0xdb, 0xe0, 0xed, 0xd3, 0x48, 0x4d, 0x2a, 0x7a, 0x60, 0x59, 0x8c, 0xa4,
+	0xb8, 0xbf, 0x14, 0xb9, 0x34, 0x79, 0xc2, 0xb3, 0xc2, 0x5e, 0x8a, 0x4c, 0x62, 0x8e, 0xba, 0x14,
+	0xcf, 0x24, 0x07, 0x7f, 0x57, 0x0c, 0x79, 0xb0, 0x08, 0x6e, 0xd4, 0x33, 0x1c, 0x6e, 0xd4, 0x0b,
+	0xde, 0x41, 0x7a, 0x93, 0x9a, 0x6e, 0xe5, 0xc4, 0x3e, 0x8d, 0x28, 0x1a, 0xbe, 0x0f, 0xdd, 0x28,
+	0xdf, 0x12, 0x22, 0x1b, 0xc6, 0x29, 0x93, 0x22, 0x33, 0xbb, 0xfd, 0x22, 0x88, 0x13, 0x24, 0x99,
+	0xd4, 0x9b, 0xb8, 0x4d, 0xb5, 0x40, 0x9e, 0xc1, 0x92, 0x32, 0x8a, 0x82, 0xad, 0xf7, 0x0a, 0x34,
+	0x15, 0x56, 0x3a, 0x61, 0xa4, 0x8a, 0xc1, 0xad, 0x33, 0x7c, 0xad, 0x19, 0xb6, 0x4f, 0x78, 0x2a,
+	0x6b, 0x1d, 0x83, 0x32, 0x12, 0x74, 0xa9, 0x16, 0x02, 0xa2, 0x03, 0x34, 0x91, 0x2c, 0x56, 0x91,
+	0x28, 0x94, 0xa2, 0x8e, 0xfc, 0xec, 0x00, 0x58, 0x87, 0x8a, 0xbc, 0x7c, 0xe2, 0x5c, 0xfd, 0x24,
+	0x58, 0xb7, 0x95, 0x37, 0xd3, 0xb2, 0x54, 0xdd, 0xd2, 0x38, 0xb5, 0x9d, 0xf1, 0x61, 0xd5, 0x19,
+	0xba, 0xa4, 0x77, 0xa7, 0x3a, 0x43, 0x5b, 0xad, 0xfa, 0xe3, 0x25, 0x74, 0x6a, 0xf8, 0xcc, 0x2e,
+	0xf9, 0xa0, 0xec, 0x12, 0x77, 0x9a, 0x12, 0x71, 0x43, 0x69, 0x7b, 0xe5, 0x05, 0x74, 0x6a, 0xf0,
+	0x4c, 0xc6, 0x75, 0xb8, 0x7d, 0x71, 0x0e, 0xed, 0x7e, 0x9f, 0x86, 0x49, 0x0c, 0xdd, 0xad, 0xa4,
+	0xc8, 0x25, 0xcf, 0x0c, 0x9d, 0xfa, 0x28, 0x68, 0xa0, 0x2c, 0x5e, 0x05, 0xcc, 0xae, 0x5f, 0x70,
+	0x1f, 0x1a, 0x2a, 0x8d, 0x7a, 0x9c, 0x2e, 0xe7, 0x58, 0x2b, 0xc9, 0x01, 0xb4, 0x36, 0xfb, 0xd1,
+	0xf3, 0x4c, 0x14, 0xe3, 0x99, 0x4e, 0xdb, 0x6f, 0xb5, 0x7b, 0xf9, 0x5b, 0xed, 0x5d, 0xfa, 0x56,
+	0xfb, 0xe5, 0xb7, 0x9a, 0xf4, 0x61, 0x59, 0xaf, 0x4a, 0x35, 0xc5, 0x37, 0x59, 0x38, 0xf6, 0x43,
+	0xea, 0xd5, 0x3e, 0xa4, 0x7d, 0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x24, 0xfd, 0xd5, 0x85, 0x65, 0xca,
+	0xf3, 0xf8, 0x35, 0x8f, 0xd2, 0x5c, 0x66, 0xc5, 0x40, 0xed, 0x24, 0xf5, 0xfe, 0x2b, 0x71, 0x68,
+	0xb2, 0xed, 0x51, 0x2d, 0x5c, 0xa7, 0xd3, 0x83, 0x47, 0xd0, 0x99, 0x9e, 0xd9, 0xcb, 0x57, 0xeb,
+	0x57, 0x82, 0x47, 0xb0, 0xd0, 0x17, 0x45, 0x36, 0x28, 0xdb, 0xb7, 0xb6, 0x27, 0xb5, 0x67, 0x5a,
+	0x4d, 0xed, 0xb5, 0xda, 0x68, 0x34, 0xe6, 0x8c, 0xc6, 0xd3, 0xa9, 0x56, 0x0a, 0x9b, 0xf8, 0xe0,
+	0x8d, 0xea, 0xc1, 0x05, 0x35, 0xbd, 0x78, 0x9b, 0xfc, 0xe4, 0xc0, 0xad, 0xba, 0x0b, 0xd7, 0x1a,
+	0xdc, 0xb2, 0x22, 0xee, 0xcc, 0x8a, 0x78, 0xb3, 0x2a, 0xe2, 0x57, 0x15, 0xa9, 0x7e, 0x13, 0x34,
+	0x6a, 0xbf, 0x09, 0xc8, 0x31, 0xdc, 0xbb, 0x54, 0xa6, 0x2d, 0x31, 0x1a, 0xab, 0x7e, 0xf8, 0x17,
+	0xe5, 0x52, 0x2b, 0x2d, 0xcb, 0x4c, 0xa1, 0xda, 0x54, 0x0b, 0xe4, 0x63, 0xb8, 0xdb, 0xe7, 0xb2,
+	0x56, 0x24, 0xdb, 0x6d, 0x6b, 0xe0, 0xed, 0xf2, 0xd3, 0x2b, 0xc2, 0x57, 0x2a, 0xf2, 0x19, 0x84,
+	0xfb, 0xe3, 0x21, 0x93, 0xfc, 0x46, 0xaf, 0x37, 0xa1, 0xb5, 0x27, 0xc6, 0x22, 0x11, 0xaf, 0x26,
+	0x73, 0xa6, 0x3e, 0x84, 0x05, 0xbd, 0xbf, 0xf5, 0x1a, 0x69, 0x53, 0x2b, 0x92, 0x3b, 0xaa, 0xa1,
+	0x07, 0x2c, 0x19, 0x14, 0x89, 0x72, 0x43, 0xfd, 0x5e, 0xcc, 0x37, 0x97, 0x7e, 0x3f, 0x5f, 0x75,
+	0xfe, 0x38, 0x5f, 0x75, 0xfe, 0x3c, 0x5f, 0x75, 0x7e, 0xf9, 0x6b, 0xf5, 0x7f, 0x87, 0x4d, 0xfc,
+	0x3f, 0xf1, 0xe4, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x46, 0x93, 0xc0, 0xc1, 0x60, 0x0c, 0x00,
+	0x00,
 }
--- a/internal/private.proto
+++ b/internal/private.proto
@ -99,6 +99,7 @@ message Node {
    string ID = 1;
    URI URI = 2;
    bool IsCoordinator = 3;
+    string State = 4;
 }

 message NodeStateMessage {
--- a/server.go
+++ b/server.go
@ -298,6 +298,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 		ID:            s.nodeID,
 		URI:           s.uri,
 		IsCoordinator: s.cluster.Coordinator == s.nodeID,
+		State:         nodeStateDown,
 	}
 	s.cluster.Node = node
 	if s.clusterDisabled {
@ -561,7 +562,10 @@ func (s *Server) receiveMessage(m Message) error {
 	case *RecalculateCaches:
 		s.holder.recalculateCaches()
 	case *NodeEvent:
-		s.cluster.ReceiveEvent(obj)
+		err := s.cluster.ReceiveEvent(obj)
+		if err != nil {
+			return errors.Wrapf(err, "cluster receiving NodeEvent %v", obj)
+		}
 	case *NodeStatus:
 		s.handleRemoteStatus(obj)
 	}
--- a/server/server.go
+++ b/server/server.go
@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
+//
 // Package server contains the `pilosa server` subcommand which runs Pilosa
 // itself. The purpose of this package is to define an easily tested Command
 // object which handles interpreting configuration and setting up all the
@ -20,6 +20,7 @@
 package server

 import (
+	"bytes"
 	"crypto/tls"
 	"io"
 	"log"
@ -336,7 +337,7 @@ func (m *Command) setupNetworking() error {
 	gossipMemberSet, err := gossip.NewMemberSet(
 		m.Config.Gossip,
 		m.API,
-		gossip.WithLogger(m.logger.Logger()),
+		gossip.WithLogOutput(&filteredWriter{logOutput: m.logOutput, v: m.Config.Verbose}),
 		gossip.WithTransport(m.gossipTransport),
 	)
 	if err != nil {
@ -407,3 +408,24 @@ func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err erro

 	return ln, nil
 }
+
+type filteredWriter struct {
+	v         bool
+	logOutput io.Writer
+}
+
+// Write forwards the write to logOutput if verbose is true, or it doesn't
+// contain [DEBUG] or [INFO]. This implementation isn't technically correct
+// since Write could be called with only part of a log line, but I don't think
+// that actually happens, so until it becomes a problem, I don't think it's
+// worth dealing with the extra complexity. (jaffee)
+func (f *filteredWriter) Write(p []byte) (n int, err error) {
+	if bytes.Contains(p, []byte("[DEBUG]")) || bytes.Contains(p, []byte("[INFO]")) {
+		if f.v {
+			return f.logOutput.Write(p)
+		}
+	} else {
+		return f.logOutput.Write(p)
+	}
+	return len(p), nil
+}