From 8db7a78a937b66ce4d48f20cd1e6cfe578a390bb Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 23 Oct 2018 11:10:24 -0500
Subject: [PATCH 01/20] unpushed WIP on cluster-tests

---
 gossip/gossip.go      | 28 ++++++++++++++++++++----
 server.go             | 16 ++++++++++----
 server/config.go      |  6 +++++
 server/server.go      |  6 +++++
 server/server_test.go |  3 ++-
 test/pilosa.go        | 51 ++++++++++++++++++++++++++++++++++++++++++-
 uri.go                | 17 +++++++++++----
 7 files changed, 113 insertions(+), 14 deletions(-)

diff --git a/gossip/gossip.go b/gossip/gossip.go
index 789ed50b0..d03016792 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -205,8 +205,18 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem
 	conf.Name = api.Node().ID
 	conf.BindAddr = api.Node().URI.Host
 	conf.BindPort = port
+	if cfg.AdvertisePort != "" {
+		port, err = strconv.Atoi(cfg.Port)
+		if err != nil {
+			return nil, fmt.Errorf("convert advertise port: %s", err)
+		}
+	}
 	conf.AdvertisePort = port
-	conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
+	if cfg.AdvertiseHost != "" {
+		conf.AdvertiseAddr = cfg.AdvertiseHost
+	} else {
+		conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
+	}
 	//
 	conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
 	conf.SuspicionMult = cfg.SuspicionMult
@@ -447,10 +457,20 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
 
 // Config holds toml-friendly memberlist configuration.
 type Config struct {
+	// Host is the host gossip will bind to. If left blank it will be set to the
+	// host from Pilosa.
+	Host string `toml:"host"`
 	// Port indicates the port to which pilosa should bind for internal state sharing.
-	Port  string   `toml:"port"`
-	Seeds []string `toml:"seeds"`
-	Key   string   `toml:"key"`
+	Port string `toml:"port"`
+	// AdvertiseHost is the hostname or IP other nodes should use to connect to
+	// this host. If left blank, the value for Host will be used. This is useful
+	// in some proxy and NAT scenarios.
+	AdvertiseHost string `toml:"advertise-host`
+	// AdvertisePort is the port other nodes will use to connect to this one.
+	// Behaves like AdvertiseHost.
+	AdvertisePort string   `toml:"advertise-port"`
+	Seeds         []string `toml:"seeds"`
+	Key           string   `toml:"key"`
 	// StreamTimeout is the timeout for establishing a stream connection with
 	// a remote node for a full state sync, and for stream read and write
 	// operations. Maps to memberlist TCPTimeout.
diff --git a/server.go b/server.go
index a4cb8fc8b..36190e315 100644
--- a/server.go
+++ b/server.go
@@ -62,6 +62,7 @@ type Server struct { // nolint: maligned
 
 	nodeID              string
 	uri                 URI
+	advertiseURI        URI
 	antiEntropyInterval time.Duration
 	metricInterval      time.Duration
 	diagnosticInterval  time.Duration
@@ -73,7 +74,7 @@ type Server struct { // nolint: maligned
 	dataDir       string
 }
 
-// TODO: have this return an interface for Holder instead of concrete object?
+// TODO (2.0): have this return an interface for Holder instead of concrete object?
 func (s *Server) Holder() *Holder {
 	return s.holder
 }
@@ -160,6 +161,13 @@ func OptServerInternalClient(c InternalClient) ServerOption {
 	}
 }
 
+func OptServerAdvertiseURI(u *URI) ServerOption {
+	return func(s *Server) error {
+		s.advertiseURI = *u
+		return nil
+	}
+}
+
 // DEPRECATED
 func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption {
 	return func(s *Server) error {
@@ -296,7 +304,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	// Set Cluster Node.
 	node := &Node{
 		ID:            s.nodeID,
-		URI:           s.uri,
+		URI:           s.advertiseURI,
 		IsCoordinator: s.cluster.Coordinator == s.nodeID,
 	}
 	s.cluster.Node = node
@@ -581,7 +589,7 @@ func (s *Server) SendSync(m Message) error {
 		node := node
 		s.logger.Printf("SendSync to: %s", node.URI)
 		// Don't forward the message to ourselves.
-		if s.uri == node.URI {
+		if s.advertiseURI == node.URI {
 			continue
 		}
 
@@ -676,7 +684,7 @@ func (s *Server) monitorDiagnostics() {
 
 	s.diagnostics.Logger = s.logger
 	s.diagnostics.SetVersion(Version)
-	s.diagnostics.Set("Host", s.uri.Host)
+	s.diagnostics.Set("Host", s.advertiseURI.Host)
 	s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ","))
 	s.diagnostics.Set("NumNodes", len(s.cluster.nodes))
 	s.diagnostics.Set("NumCPU", runtime.NumCPU())
diff --git a/server/config.go b/server/config.go
index d255e4bb6..210a2e298 100644
--- a/server/config.go
+++ b/server/config.go
@@ -36,9 +36,15 @@ type Config struct {
 	// DataDir is the directory where Pilosa stores both indexed data and
 	// running state such as cluster topology information.
 	DataDir string `toml:"data-dir"`
+
 	// Bind is the host:port on which Pilosa will listen.
 	Bind string `toml:"bind"`
 
+	// Advertise is the host:port that this node will report as its address to
+	// others. If left blank (the default), this will be set to the bind address
+	// once it is listening.
+	Advertise string `toml:"advertise"`
+
 	// MaxWritesPerRequest limits the number of mutating commands that can be in
 	// a single request to the server. This includes Set, Clear,
 	// SetRowAttrs & SetColumnAttrs.
diff --git a/server/server.go b/server/server.go
index 010eb2f8a..8521abdec 100644
--- a/server/server.go
+++ b/server/server.go
@@ -248,6 +248,11 @@ func (m *Command) SetupServer() error {
 		uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port))
 	}
 
+	advertURI, err := pilosa.NewURIFromAddressWithDefault(m.Config.Advertise, uri)
+	if err != nil {
+		return errors.Wrapf(err, "processing avertise address '%s'", m.Config.Advertise)
+	}
+
 	c := http.GetHTTPClient(TLSConfig)
 
 	// Primary store configuration is handled automatically now.
@@ -276,6 +281,7 @@ func (m *Command) SetupServer() error {
 		pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
 		pilosa.OptServerStatsClient(statsClient),
 		pilosa.OptServerURI(uri),
+		pilosa.OptServerAdvertiseURI(advertURI),
 		pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
 		pilosa.OptServerPrimaryTranslateStoreFunc(http.NewTranslateStore),
 		pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
diff --git a/server/server_test.go b/server/server_test.go
index aa202a54c..7a5e85572 100644
--- a/server/server_test.go
+++ b/server/server_test.go
@@ -405,7 +405,6 @@ func TestClusteringNodesReplica1(t *testing.T) {
 	// Create new main with the same config.
 	config := cluster[2].Command.Config
 	config.Translation.MapSize = 100000
-	// config.Bind = cluster[2].API.Node().URI.HostPort()
 
 	// this isn't necessary, but makes the test run way faster
 	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
@@ -413,6 +412,8 @@ func TestClusteringNodesReplica1(t *testing.T) {
 	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr)
 	cluster[2].Command.Config = config
 
+	time.Sleep(time.Second * 40)
+
 	// Run new program.
 	if err := cluster[2].Start(); err != nil {
 		t.Fatalf("restarting node 2: %v", err)
diff --git a/test/pilosa.go b/test/pilosa.go
index 27331d53f..46424e09d 100644
--- a/test/pilosa.go
+++ b/test/pilosa.go
@@ -19,11 +19,13 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	gohttp "net/http"
 	"os"
 	"path"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -31,8 +33,37 @@ import (
 	"github.com/pilosa/pilosa/http"
 	"github.com/pilosa/pilosa/server"
 	"github.com/pkg/errors"
+
+	"github.com/Shopify/toxiproxy"
+	tox "github.com/Shopify/toxiproxy/client"
 )
 
+type portAllocator struct {
+	port *uint32
+}
+
+var ports *portAllocator
+
+// Next generates a new port and returns it
+func (n *portAllocator) Next() (nextPort uint32) {
+	return atomic.AddUint32(n.port, 1)
+}
+
+var proxy string
+
+func init() {
+	r := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
+	p := uint32(r.Uint32()%40000 + 20000)
+	ports = &portAllocator{
+		port: &p,
+	}
+
+	tport := strconv.Itoa(int(ports.Next()))
+	proxy = "localhost:" + tport
+	go toxiproxy.NewServer().Listen("localhost", tport)
+
+}
+
 ////////////////////////////////////////////////////////////////////////////////////
 // Command represents a test wrapper for server.Command.
 type Command struct {
@@ -232,6 +263,8 @@ func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu
 
 // newCluster creates a new cluster
 func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
+	tclient := tox.NewClient(proxy)
+
 	if size == 0 {
 		return nil, errors.New("cluster must contain at least one node")
 	}
@@ -245,8 +278,24 @@ func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
 		if len(opts) > 0 {
 			commandOpts = opts[i%len(opts)]
 		}
+		aport := strconv.Itoa(int(ports.Next()))
+		name := "node" + strconv.Itoa(i)
 		m := NewCommandNode(i == 0, commandOpts...)
-		err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte("node"+strconv.Itoa(i)), 0600)
+		m.Config.Bind = "localhost:" + strconv.Itoa(int(ports.Next()))
+		m.Config.Advertise = "localhost:" + aport
+		p, err := tclient.CreateProxy(name+aport, m.Config.Advertise, m.Config.Bind)
+		if err != nil {
+			return nil, errors.Wrap(err, "setting up toxiproxy")
+		}
+		m.Config.Gossip.Port = strconv.Itoa(int(ports.Next()))
+		aport = strconv.Itoa(int(ports.Next()))
+		m.Config.Gossip.AdvertisePort = aport
+		p, err = tclient.CreateProxy(name+"-gossip"+aport, "localhost:"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.Port)
+		if err != nil {
+			return nil, errors.Wrap(err, "setting up toxiproxy for gossip")
+		}
+		fmt.Println(p)
+		err = ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name), 0600)
 		if err != nil {
 			return nil, errors.Wrap(err, "writing node id")
 		}
diff --git a/uri.go b/uri.go
index 231457691..7cf985f0b 100644
--- a/uri.go
+++ b/uri.go
@@ -82,6 +82,10 @@ func NewURIFromAddress(address string) (*URI, error) {
 	return parseAddress(address)
 }
 
+func NewURIFromAddressWithDefault(address string, base *URI) (*URI, error) {
+	return parseAddressWithDefault(address, base)
+}
+
 // setScheme sets the scheme of this URI.
 func (u *URI) setScheme(scheme string) error {
 	m := schemeRegexp.FindStringSubmatch(scheme)
@@ -154,20 +158,20 @@ func (u URI) Type() string {
 	return "URI"
 }
 
-func parseAddress(address string) (uri *URI, err error) {
+func parseAddressWithDefault(address string, def *URI) (uri *URI, err error) {
 	m := addressRegexp.FindStringSubmatch(address)
 	if m == nil {
 		return nil, errors.New("invalid address")
 	}
-	scheme := "http"
+	scheme := def.Scheme
 	if m[2] != "" {
 		scheme = m[2]
 	}
-	host := "localhost"
+	host := def.Host
 	if m[3] != "" {
 		host = m[3]
 	}
-	var port = 10101
+	var port = int(def.Port)
 	if m[5] != "" {
 		port, err = strconv.Atoi(m[5])
 		if err != nil {
@@ -185,6 +189,11 @@ func parseAddress(address string) (uri *URI, err error) {
 	return uri, nil
 }
 
+func parseAddress(address string) (uri *URI, err error) {
+	u, err := parseAddressWithDefault(address, defaultURI())
+	return u, err
+}
+
 // MarshalJSON marshals URI into a JSON-encoded byte slice.
 func (u *URI) MarshalJSON() ([]byte, error) {
 	var output struct {

From f643487ce086beb616a9ac00a500eb26df6f94f2 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Wed, 24 Oct 2018 16:55:39 -0500
Subject: [PATCH 02/20] add initial UDP proxy code to support
 proxying/partitioning memberlist

---
 internal/udproxy/udproxy.go      | 145 +++++++++++++++++++++++++++++++
 internal/udproxy/udproxy_test.go |  63 ++++++++++++++
 2 files changed, 208 insertions(+)
 create mode 100644 internal/udproxy/udproxy.go
 create mode 100644 internal/udproxy/udproxy_test.go

diff --git a/internal/udproxy/udproxy.go b/internal/udproxy/udproxy.go
new file mode 100644
index 000000000..fa30b22a5
--- /dev/null
+++ b/internal/udproxy/udproxy.go
@@ -0,0 +1,145 @@
+package udproxy
+
+import (
+	"bytes"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+type Proxy struct {
+	upstreamAddr *net.UDPAddr
+	conn         *net.UDPConn
+
+	drop     bool
+	dropLock sync.Mutex
+
+	quit chan struct{}
+	eg   errgroup.Group
+	// map from client address to upstream connection. We must maintain a
+	// separate connection to upstream for each client connection so that we can
+	// differentiate data sent back from upstream.
+	upstreams map[*net.UDPAddr]*net.UDPConn
+	// TODO - need to track a per-connection timeout so that "upstreams" doesn't
+	// grow indefinitely.
+}
+
+func New(listenIP string, listenPort int, upstreamIP string, upstreamPort int) (*Proxy, error) {
+	uc, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP(listenIP), Port: listenPort})
+	if err != nil {
+		return nil, errors.Wrap(err, "listening")
+	}
+	p := &Proxy{
+		conn:         uc,
+		upstreamAddr: &net.UDPAddr{IP: net.ParseIP(upstreamIP), Port: upstreamPort},
+		quit:         make(chan struct{}),
+		upstreams:    make(map[*net.UDPAddr]*net.UDPConn),
+	}
+	if p.upstreamAddr.IP == nil {
+		return nil, errors.Errorf("unable to parse upstream ip '%s'", upstreamIP)
+	}
+	p.eg.Go(p.run)
+	return p, nil
+}
+
+func (p *Proxy) Drop() {
+	p.dropLock.Lock()
+	p.drop = true
+	p.dropLock.Unlock()
+}
+
+func (p *Proxy) Undrop() {
+	p.dropLock.Lock()
+	p.drop = false
+	p.dropLock.Unlock()
+}
+
+func (p *Proxy) dropping() bool {
+	p.dropLock.Lock()
+	d := p.drop
+	p.dropLock.Unlock()
+	return d
+}
+
+func (p *Proxy) run() error {
+	buf := make([]byte, 65507)
+	for {
+		select {
+		case <-p.quit:
+			return nil
+		default:
+		}
+		err := p.conn.SetReadDeadline(time.Now().Add(time.Millisecond * 10))
+		if err != nil {
+			return errors.Wrap(err, "setting read deadline (run)")
+		}
+		n, addr, err := p.conn.ReadFromUDP(buf)
+		if err, ok := err.(net.Error); ok && err.Timeout() {
+			continue
+		} else if err != nil {
+			return errors.Wrap(err, "reading from udp conn")
+		}
+		upConn := p.upstreams[addr]
+		if upConn == nil {
+			p.upstreams[addr], err = net.DialUDP("udp", &net.UDPAddr{}, p.upstreamAddr)
+			if err != nil {
+				return errors.Wrap(err, "creating new connection to upstream")
+			}
+			p.eg.Go(func() error {
+				return p.proxyBack(addr, p.upstreams[addr])
+			})
+			upConn = p.upstreams[addr]
+		}
+		if !p.dropping() {
+			_, err = io.Copy(upConn, bytes.NewBuffer(buf[:n]))
+			if err != nil {
+				return errors.Wrap(err, "writing to upstream conn")
+			}
+		}
+	}
+}
+
+func (p *Proxy) proxyBack(to *net.UDPAddr, from *net.UDPConn) error {
+	buf := make([]byte, 65507)
+	for {
+		select {
+		case <-p.quit:
+			return nil
+		default:
+		}
+		err := from.SetReadDeadline(time.Now().Add(time.Millisecond * 10))
+		if err != nil {
+			return errors.Wrap(err, "setting read deadline (proxyBack)")
+		}
+		n, _, err := from.ReadFromUDP(buf)
+		if err, ok := err.(net.Error); ok && err.Timeout() {
+			continue
+		} else if err != nil {
+			return errors.Wrap(err, "reading from upstream")
+		}
+		if !p.dropping() {
+			_, err = io.Copy(addrWriter{c: p.conn, a: to}, bytes.NewBuffer(buf[:n]))
+			if err != nil {
+				return errors.Wrap(err, "writing back to client")
+			}
+		}
+	}
+}
+
+type addrWriter struct {
+	c *net.UDPConn
+	a *net.UDPAddr
+}
+
+func (a addrWriter) Write(b []byte) (n int, err error) {
+	return a.c.WriteTo(b, a.a)
+}
+
+func (p *Proxy) Close() error {
+	close(p.quit)
+	return p.eg.Wait()
+}
diff --git a/internal/udproxy/udproxy_test.go b/internal/udproxy/udproxy_test.go
new file mode 100644
index 000000000..edb915d1f
--- /dev/null
+++ b/internal/udproxy/udproxy_test.go
@@ -0,0 +1,63 @@
+package udproxy_test
+
+import (
+	"net"
+	"testing"
+
+	"github.com/pilosa/pilosa/internal/udproxy"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+func TestUDProxy(t *testing.T) {
+	p, err := udproxy.New("127.0.0.1", 12345, "127.0.0.1", 12346)
+	if err != nil {
+		t.Fatalf("creating proxy: %v", err)
+	}
+
+	uc, err := net.ListenUDP("udp", &net.UDPAddr{Port: 12346})
+	if err != nil {
+		t.Fatalf("listening udp upstream: %v", err)
+	}
+
+	resp := make([]byte, 8)
+	eg := errgroup.Group{}
+	eg.Go(func() error {
+		conn, err := net.DialUDP("udp", nil, &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 12345})
+		if err != nil {
+			t.Fatalf("connecting to proxy: %v", err)
+		}
+		_, err = conn.Write([]byte("hello!"))
+		if err != nil {
+			return errors.Wrap(err, "writing to proxy")
+		}
+		_, err = conn.Read(resp)
+		if err != nil {
+			return errors.Wrap(err, "reading from proxy")
+		}
+		return nil
+	})
+
+	req := make([]byte, 10)
+	_, addr, err := uc.ReadFrom(req)
+	if err != nil {
+		t.Fatalf("upstream reading from proxy: %v", err)
+	}
+	if string(req[:6]) != "hello!" {
+		t.Fatalf("got unexpected request %s", req)
+	}
+	_, err = uc.WriteTo([]byte("goodbye"), addr)
+	if err != nil {
+		t.Fatalf("writing response: %v", err)
+	}
+	eg.Wait()
+	if string(resp[:7]) != "goodbye" {
+		t.Fatalf("got unexpected response '%v", resp)
+	}
+	err = p.Close()
+	if err != nil {
+		t.Fatalf("err closing proxy: '%v'", err)
+	}
+}
+
+// TODO test dropping

From 28591b4a916659166a6345ae2ad373f7d9d23572 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Wed, 24 Oct 2018 16:56:32 -0500
Subject: [PATCH 03/20] WIP: figure out why config.Gossip.Port is 0 after
 cluster start

---
 server/server_test.go |  2 ++
 test/pilosa.go        | 18 +++++++++++-------
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/server/server_test.go b/server/server_test.go
index 7a5e85572..3900a7faa 100644
--- a/server/server_test.go
+++ b/server/server_test.go
@@ -407,7 +407,9 @@ func TestClusteringNodesReplica1(t *testing.T) {
 	config.Translation.MapSize = 100000
 
 	// this isn't necessary, but makes the test run way faster
+	fmt.Println("!!!!!!!!!!", config.Gossip.Port)
 	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
+	fmt.Println("!!!!!!!!!!!!!!!!", config.Gossip.Port)
 
 	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr)
 	cluster[2].Command.Config = config
diff --git a/test/pilosa.go b/test/pilosa.go
index 46424e09d..901e8ffd2 100644
--- a/test/pilosa.go
+++ b/test/pilosa.go
@@ -34,8 +34,8 @@ import (
 	"github.com/pilosa/pilosa/server"
 	"github.com/pkg/errors"
 
-	"github.com/Shopify/toxiproxy"
-	tox "github.com/Shopify/toxiproxy/client"
+	"github.com/jaffee/toxiproxy"
+	tox "github.com/jaffee/toxiproxy/client"
 )
 
 type portAllocator struct {
@@ -283,18 +283,22 @@ func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
 		m := NewCommandNode(i == 0, commandOpts...)
 		m.Config.Bind = "localhost:" + strconv.Itoa(int(ports.Next()))
 		m.Config.Advertise = "localhost:" + aport
-		p, err := tclient.CreateProxy(name+aport, m.Config.Advertise, m.Config.Bind)
+		_, err := tclient.CreateProxy(name+aport, m.Config.Advertise, m.Config.Bind)
 		if err != nil {
 			return nil, errors.Wrap(err, "setting up toxiproxy")
 		}
-		m.Config.Gossip.Port = strconv.Itoa(int(ports.Next()))
-		aport = strconv.Itoa(int(ports.Next()))
+		gossipBindPort := strconv.Itoa(int(ports.Next()))
+		m.Config.Gossip.Port = gossipBindPort
+		gossipAdvertPort := strconv.Itoa(int(ports.Next()))
 		m.Config.Gossip.AdvertisePort = aport
-		p, err = tclient.CreateProxy(name+"-gossip"+aport, "localhost:"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.Port)
+		_, err = tclient.CreateProxy(name+"-gossip"+gossipAdvertPort, "localhost:"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.Port)
 		if err != nil {
 			return nil, errors.Wrap(err, "setting up toxiproxy for gossip")
 		}
-		fmt.Println(p)
+		_, err = tclient.CreateProxy(name+"-gossipudp"+gossipAdvertPort, "localhost:"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.Port, tox.CreateWithProtocol("udp"))
+		if err != nil {
+			return nil, errors.Wrap(err, "setting up toxiproxy for udp gossip")
+		}
 		err = ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name), 0600)
 		if err != nil {
 			return nil, errors.Wrap(err, "writing node id")

From af7ece74fd031be8589c4e9c158fa06df8948488 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Fri, 26 Oct 2018 09:17:26 -0500
Subject: [PATCH 04/20] test drop and undrop in udproxy.

---
 internal/udproxy/udproxy.go      | 10 ++++++++--
 internal/udproxy/udproxy_test.go | 20 ++++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/internal/udproxy/udproxy.go b/internal/udproxy/udproxy.go
index fa30b22a5..a79c0121f 100644
--- a/internal/udproxy/udproxy.go
+++ b/internal/udproxy/udproxy.go
@@ -47,12 +47,18 @@ func New(listenIP string, listenPort int, upstreamIP string, upstreamPort int) (
 }
 
 func (p *Proxy) Drop() {
+	// sleep to attempt to ensure all traffic that was supposed to pass, did
+	// pass.
+	time.Sleep(time.Millisecond * 3)
 	p.dropLock.Lock()
 	p.drop = true
 	p.dropLock.Unlock()
 }
 
 func (p *Proxy) Undrop() {
+	// this sleep is a cheap attempt to ensure everything that was supposed to
+	// be dropped was.
+	time.Sleep(time.Millisecond * 3)
 	p.dropLock.Lock()
 	p.drop = false
 	p.dropLock.Unlock()
@@ -73,7 +79,7 @@ func (p *Proxy) run() error {
 			return nil
 		default:
 		}
-		err := p.conn.SetReadDeadline(time.Now().Add(time.Millisecond * 10))
+		err := p.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
 		if err != nil {
 			return errors.Wrap(err, "setting read deadline (run)")
 		}
@@ -111,7 +117,7 @@ func (p *Proxy) proxyBack(to *net.UDPAddr, from *net.UDPConn) error {
 			return nil
 		default:
 		}
-		err := from.SetReadDeadline(time.Now().Add(time.Millisecond * 10))
+		err := from.SetReadDeadline(time.Now().Add(time.Millisecond))
 		if err != nil {
 			return errors.Wrap(err, "setting read deadline (proxyBack)")
 		}
diff --git a/internal/udproxy/udproxy_test.go b/internal/udproxy/udproxy_test.go
index edb915d1f..be746ea72 100644
--- a/internal/udproxy/udproxy_test.go
+++ b/internal/udproxy/udproxy_test.go
@@ -35,6 +35,16 @@ func TestUDProxy(t *testing.T) {
 		if err != nil {
 			return errors.Wrap(err, "reading from proxy")
 		}
+		p.Drop()
+		_, err = conn.Write([]byte("hello2"))
+		if err != nil {
+			return errors.Wrap(err, "writing to dropping proxy")
+		}
+		p.Undrop()
+		_, err = conn.Write([]byte("hello3"))
+		if err != nil {
+			return errors.Wrap(err, "writing to undropping proxy")
+		}
 		return nil
 	})
 
@@ -50,10 +60,20 @@ func TestUDProxy(t *testing.T) {
 	if err != nil {
 		t.Fatalf("writing response: %v", err)
 	}
+
+	_, _, err = uc.ReadFrom(req)
+	if err != nil {
+		t.Fatalf("upstream reading from proxy: %v", err)
+	}
+	if string(req[:6]) != "hello3" {
+		t.Fatalf("got unexpected request %s", req)
+	}
+
 	eg.Wait()
 	if string(resp[:7]) != "goodbye" {
 		t.Fatalf("got unexpected response '%v", resp)
 	}
+
 	err = p.Close()
 	if err != nil {
 		t.Fatalf("err closing proxy: '%v'", err)

From 62c1bfe90ecfd2b074f02f36a31f63125f54432b Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Fri, 26 Oct 2018 16:13:20 -0500
Subject: [PATCH 05/20] implement MustNewClusterWithProxy and drop/undrop for
 partitioning

---
 server/server_test.go |  19 ++++++--
 test/pilosa.go        | 111 +++++++++++++++++++++++++++++++-----------
 2 files changed, 97 insertions(+), 33 deletions(-)

diff --git a/server/server_test.go b/server/server_test.go
index 3900a7faa..f30c02774 100644
--- a/server/server_test.go
+++ b/server/server_test.go
@@ -379,7 +379,11 @@ func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
 
 func TestClusteringNodesReplica1(t *testing.T) {
-	cluster := test.MustRunCluster(t, 3)
+	cluster := test.MustNewCluster(t, 3)
+	err := cluster.Start()
+	if err != nil {
+		t.Fatalf("starting cluster: %v", err)
+	}
 	defer cluster.Close()
 
 	var wait = true
@@ -407,15 +411,11 @@ func TestClusteringNodesReplica1(t *testing.T) {
 	config.Translation.MapSize = 100000
 
 	// this isn't necessary, but makes the test run way faster
-	fmt.Println("!!!!!!!!!!", config.Gossip.Port)
 	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
-	fmt.Println("!!!!!!!!!!!!!!!!", config.Gossip.Port)
 
 	cluster[2].Command = server.NewCommand(cluster[2].Stdin, cluster[2].Stdout, cluster[2].Stderr)
 	cluster[2].Command.Config = config
 
-	time.Sleep(time.Second * 40)
-
 	// Run new program.
 	if err := cluster[2].Start(); err != nil {
 		t.Fatalf("restarting node 2: %v", err)
@@ -696,3 +696,12 @@ func TestClusterQueriesAfterRestart(t *testing.T) {
 }
 
 // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.
+
+func TestClusterPartitioning(t *testing.T) {
+	cluster := test.MustNewClusterWithProxy(t, 3)
+	err := cluster.Start()
+	if err != nil {
+		t.Fatalf("starting cluster with proxy: %v", err)
+	}
+
+}
diff --git a/test/pilosa.go b/test/pilosa.go
index 901e8ffd2..cf197e163 100644
--- a/test/pilosa.go
+++ b/test/pilosa.go
@@ -31,11 +31,12 @@ import (
 
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/http"
+	"github.com/pilosa/pilosa/internal/udproxy"
 	"github.com/pilosa/pilosa/server"
 	"github.com/pkg/errors"
 
-	"github.com/jaffee/toxiproxy"
-	tox "github.com/jaffee/toxiproxy/client"
+	"github.com/Shopify/toxiproxy"
+	tox "github.com/Shopify/toxiproxy/client"
 )
 
 type portAllocator struct {
@@ -69,9 +70,49 @@ func init() {
 type Command struct {
 	*server.Command
 
+	proxies commandProxies
+
 	commandOptions []server.CommandOption
 }
 
+// Drop uses the proxies to drop all network traffic to/from this node.
+func (com Command) Drop(tb testing.TB) {
+	if com.proxies.http == nil {
+		tb.Fatal("can't drop traffic if cluster wasn't created with proxy support.")
+	}
+	err := com.proxies.http.Disable()
+	if err != nil {
+		tb.Fatalf("disabling http proxy: %v", err)
+	}
+	err = com.proxies.memberTCP.Disable()
+	if err != nil {
+		tb.Fatalf("disabling memberlist tcp proxy: %v", err)
+	}
+	com.proxies.memberUDP.Drop()
+}
+
+// Undrop starts forwarding traffic to/from this node after a previous drop request.
+func (com Command) Undrop(tb testing.TB) {
+	if com.proxies.http == nil {
+		tb.Fatal("can't drop traffic if cluster wasn't created with proxy support.")
+	}
+	err := com.proxies.http.Enable()
+	if err != nil {
+		tb.Fatalf("enabling http proxy: %v", err)
+	}
+	err = com.proxies.memberTCP.Enable()
+	if err != nil {
+		tb.Fatalf("enabling memberlist tcp proxy: %v", err)
+	}
+	com.proxies.memberUDP.Undrop()
+}
+
+type commandProxies struct {
+	http      *tox.Proxy
+	memberTCP *tox.Proxy
+	memberUDP *udproxy.Proxy
+}
+
 func OptAllowedOrigins(origins []string) server.CommandOption {
 	return func(m *server.Command) error {
 		m.Config.Handler.AllowedOrigins = origins
@@ -232,7 +273,6 @@ type Cluster []*Command
 func (c Cluster) Start() error {
 	var gossipSeeds = make([]string, len(c))
 	for i, cc := range c {
-		cc.Config.Gossip.Port = "0"
 		cc.Config.Gossip.Seeds = gossipSeeds[:i]
 		if err := cc.Start(); err != nil {
 			return errors.Wrapf(err, "starting server %d", i)
@@ -254,7 +294,17 @@ func (c Cluster) Close() error {
 
 // MustNewCluster creates a new cluster
 func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
-	c, err := newCluster(size, opts...)
+	c, err := newCluster(size, false, opts...)
+	if err != nil {
+		tb.Fatalf("new cluster: %v", err)
+	}
+	return c
+}
+
+// MustNewClusterWithProxy returns a test cluster which has all connections
+// going through a proxy to allow for testing network partitions.
+func MustNewClusterWithProxy(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
+	c, err := newCluster(size, true, opts...)
 	if err != nil {
 		tb.Fatalf("new cluster: %v", err)
 	}
@@ -262,9 +312,7 @@ func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu
 }
 
 // newCluster creates a new cluster
-func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
-	tclient := tox.NewClient(proxy)
-
+func newCluster(size int, withproxy bool, opts ...[]server.CommandOption) (cluster Cluster, err error) {
 	if size == 0 {
 		return nil, errors.New("cluster must contain at least one node")
 	}
@@ -272,33 +320,40 @@ func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
 		return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
 	}
 
-	cluster := make(Cluster, size)
+	cluster = make(Cluster, size)
 	for i := 0; i < size; i++ {
 		var commandOpts []server.CommandOption
 		if len(opts) > 0 {
 			commandOpts = opts[i%len(opts)]
 		}
-		aport := strconv.Itoa(int(ports.Next()))
 		name := "node" + strconv.Itoa(i)
 		m := NewCommandNode(i == 0, commandOpts...)
 		m.Config.Bind = "localhost:" + strconv.Itoa(int(ports.Next()))
-		m.Config.Advertise = "localhost:" + aport
-		_, err := tclient.CreateProxy(name+aport, m.Config.Advertise, m.Config.Bind)
-		if err != nil {
-			return nil, errors.Wrap(err, "setting up toxiproxy")
-		}
-		gossipBindPort := strconv.Itoa(int(ports.Next()))
-		m.Config.Gossip.Port = gossipBindPort
-		gossipAdvertPort := strconv.Itoa(int(ports.Next()))
-		m.Config.Gossip.AdvertisePort = aport
-		_, err = tclient.CreateProxy(name+"-gossip"+gossipAdvertPort, "localhost:"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.Port)
-		if err != nil {
-			return nil, errors.Wrap(err, "setting up toxiproxy for gossip")
-		}
-		_, err = tclient.CreateProxy(name+"-gossipudp"+gossipAdvertPort, "localhost:"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.Port, tox.CreateWithProtocol("udp"))
-		if err != nil {
-			return nil, errors.Wrap(err, "setting up toxiproxy for udp gossip")
+		gossipBindPort := int(ports.Next())
+		m.Config.Gossip.Port = strconv.Itoa(gossipBindPort)
+
+		if withproxy {
+			tclient := tox.NewClient(proxy)
+
+			aport := strconv.Itoa(int(ports.Next()))
+			m.Config.Advertise = "localhost:" + aport
+			m.proxies.http, err = tclient.CreateProxy(name+aport, m.Config.Advertise, m.Config.Bind)
+			if err != nil {
+				return nil, errors.Wrap(err, "setting up toxiproxy")
+			}
+			gossipAdvertPort := int(ports.Next())
+			m.Config.Gossip.AdvertisePort = strconv.Itoa(gossipAdvertPort)
+			m.proxies.memberTCP, err = tclient.CreateProxy(name+"-gossip"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.Port)
+			if err != nil {
+				return nil, errors.Wrap(err, "setting up toxiproxy for gossip")
+			}
+			m.proxies.memberUDP, err = udproxy.New("127.0.0.1", gossipAdvertPort, "127.0.0.1", gossipBindPort)
+			if err != nil {
+				return nil, errors.Wrap(err, "setting up proxy for udp gossip")
+			}
+
 		}
+
 		err = ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name), 0600)
 		if err != nil {
 			return nil, errors.Wrap(err, "writing node id")
@@ -310,8 +365,8 @@ func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
 }
 
 // runCluster creates and starts a new cluster
-func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
-	cluster, err := newCluster(size, opts...)
+func runCluster(size int, withproxy bool, opts ...[]server.CommandOption) (Cluster, error) {
+	cluster, err := newCluster(size, withproxy, opts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "new cluster")
 	}
@@ -323,7 +378,7 @@ func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
 
 // MustRunCluster creates and starts a new cluster
 func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
-	c, err := runCluster(size, opts...)
+	c, err := runCluster(size, false, opts...)
 	if err != nil {
 		tb.Fatalf("run cluster: %v", err)
 	}

From d86e9ed3ea11aa3d87963cd044d0ed0955a849c4 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Fri, 2 Nov 2018 14:41:42 -0500
Subject: [PATCH 06/20] add docker based cluster tests, remove proxy stuff, fix
 advertise

---
 Dockerfile-withgo                        |  22 ++++
 Gopkg.lock                               |  11 +-
 ctl/server.go                            |   1 +
 internal/clustertests/Dockerfile         |   3 +
 internal/clustertests/cluster_test.go    | 138 +++++++++++++++++++++
 internal/clustertests/docker-compose.yml |  53 ++++++++
 internal/udproxy/udproxy.go              | 151 -----------------------
 internal/udproxy/udproxy_test.go         |  83 -------------
 server/server_test.go                    |  15 +--
 test/pilosa.go                           | 124 ++-----------------
 10 files changed, 236 insertions(+), 365 deletions(-)
 create mode 100644 Dockerfile-withgo
 create mode 100644 internal/clustertests/Dockerfile
 create mode 100644 internal/clustertests/cluster_test.go
 create mode 100644 internal/clustertests/docker-compose.yml
 delete mode 100644 internal/udproxy/udproxy.go
 delete mode 100644 internal/udproxy/udproxy_test.go

diff --git a/Dockerfile-withgo b/Dockerfile-withgo
new file mode 100644
index 000000000..5cb11e47b
--- /dev/null
+++ b/Dockerfile-withgo
@@ -0,0 +1,22 @@
+FROM golang:1.11
+
+LABEL maintainer "dev@pilosa.com"
+
+COPY . /go/src/github.com/pilosa/pilosa/
+
+RUN cd /go/src/github.com/pilosa/pilosa \
+    && CGO_ENABLED=0 make install-dep install FLAGS="-a"
+
+RUN wget https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 -O /pumba
+RUN chmod +x /pumba
+
+RUN cp /go/bin/pilosa /pilosa
+
+COPY LICENSE /LICENSE
+COPY NOTICE /NOTICE
+
+EXPOSE 10101
+VOLUME /data
+
+ENTRYPOINT ["bash", "-c"]
+CMD ["pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
diff --git a/Gopkg.lock b/Gopkg.lock
index 4a6068840..280f6d056 100644
--- a/Gopkg.lock
+++ b/Gopkg.lock
@@ -189,6 +189,15 @@
   revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
   version = "v1.2.0"
 
+[[projects]]
+  name = "github.com/pilosa/go-pilosa"
+  packages = [
+    ".",
+    "gopilosa_pbuf"
+  ]
+  revision = "4e7807f5ad779407936744057cd17332046b6c3c"
+  version = "v1.1.0"
+
 [[projects]]
   name = "github.com/pkg/errors"
   packages = ["."]
@@ -324,6 +333,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "8290156ce8b4066c46ab83d743f4c81df0a17e148415bb1ee8409a51ac4c3ba4"
+  inputs-digest = "be318fa4f2a72e7e849b2faff1ce9300deaaf2d24cf76100f4b538abed86295b"
   solver-name = "gps-cdcl"
   solver-version = 1
diff --git a/ctl/server.go b/ctl/server.go
index 7f384ec7b..90766c498 100644
--- a/ctl/server.go
+++ b/ctl/server.go
@@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
+	flags.StringVarP(&srv.Config.Advertise, "advertise", "a", "", "Address to broadcast to other hosts and clients to be contacted on.")
 	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
 	flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
diff --git a/internal/clustertests/Dockerfile b/internal/clustertests/Dockerfile
new file mode 100644
index 000000000..ecbfa7f21
--- /dev/null
+++ b/internal/clustertests/Dockerfile
@@ -0,0 +1,3 @@
+FROM ptest
+
+COPY . /go/src/github.com/pilosa/pilosa/internal/clustertests
diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go
new file mode 100644
index 000000000..e63b95d0b
--- /dev/null
+++ b/internal/clustertests/cluster_test.go
@@ -0,0 +1,138 @@
+package clustertest
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/pilosa/go-pilosa"
+	pi "github.com/pilosa/pilosa"
+)
+
+func TestLongPauses(t *testing.T) {
+	t.Skip()
+	cli := getPilosaClient(t)
+
+	idx := pilosa.NewIndex("testidx")
+	err := cli.CreateIndex(idx)
+	if err != nil {
+		t.Fatalf("creating index: %v", err)
+	}
+	f := idx.Field("testf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
+	err = cli.CreateField(f)
+	if err != nil {
+		t.Fatalf("creating field: %v", err)
+	}
+
+	data := make([]pilosa.Column, 1000)
+	for i := range data {
+		data[i].RowID = 0
+		data[i].ColumnID = uint64((i/10)*pi.ShardWidth + i%10)
+	}
+
+	err = cli.ImportField(f, &colIterator{cols: data}, pilosa.OptImportBatchSize(1000))
+	if err != nil {
+		t.Fatalf("importing: %v", err)
+	}
+
+	r, err := cli.Query(idx.Count(f.Row(0)))
+	if err != nil {
+		t.Fatalf("count querying: %v", err)
+	}
+	if r.Result().Count() != 1000 {
+		t.Fatalf("count after import is %d", r.Result().Count())
+	}
+
+	pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "30s")
+	pcmd.Stdout = os.Stdout
+	pcmd.Stderr = os.Stderr
+	fmt.Println("pausing pilosa3 for 30s")
+	err = pcmd.Start()
+	if err != nil {
+		t.Fatalf("starting pumba command: %v", err)
+	}
+	err = pcmd.Wait()
+	if err != nil {
+		t.Fatalf("waiting on pumba pause cmd: %v", err)
+	}
+	fmt.Println("done with pause, waiting for stability")
+	time.Sleep(time.Second * 400)
+	fmt.Println("done waiting")
+
+	r, err = cli.Query(idx.Count(f.Row(0)))
+	if err != nil {
+		t.Fatalf("count querying: %v", err)
+	}
+	if r.Result().Count() != 1000 {
+		t.Fatalf("count after import is %d", r.Result().Count())
+	}
+}
+
+// Utils
+
+func getPilosaClient(t *testing.T) *pilosa.Client {
+	cli, err := pilosa.NewClient("pilosa1:10101")
+	if err != nil {
+		time.Sleep(time.Millisecond * 40)
+	}
+	time.Sleep(time.Second * 2)
+	// start := time.Now()
+	// for i := 0; true; i++ {
+	// 	s, err := cli.Status()
+	// 	if i > 800 {
+	// 		t.Fatalf("couldn't connect to cluster after %d attempts and %v: state: %s, err: %v", i, time.Since(start), s.State, err)
+	// 	}
+	// 	if err != nil {
+	// 		time.Sleep(time.Millisecond * 40)
+	// 		continue
+	// 	}
+	// 	if s.State == "NORMAL" {
+	// 		break
+	// 	} else {
+	// 		time.Sleep(time.Millisecond * 40)
+	// 	}
+	// }
+
+	return cli
+}
+
+type colIterator struct {
+	cols []pilosa.Column
+	i    uint
+}
+
+func (c *colIterator) NextRecord() (pilosa.Record, error) {
+	if int(c.i) >= len(c.cols) {
+		return nil, io.EOF
+	}
+	c.i++
+	return c.cols[c.i-1], nil
+}
+
+func TestColIterator(t *testing.T) {
+	data := make([]pilosa.Column, 3)
+	for i := range data {
+		data[i].RowID = 0
+		data[i].ColumnID = uint64((i/10)*pi.ShardWidth + i%10)
+	}
+
+	ci := colIterator{cols: data}
+	col := pilosa.Column{}
+	if rec, err := ci.NextRecord(); rec != col {
+		t.Fatalf("first record wrong: %v, err: %v", rec, err)
+	}
+	col.ColumnID = 1
+	if rec, err := ci.NextRecord(); rec != col || err != nil {
+		t.Fatalf("second record wrong: %v, err: %v", rec, err)
+	}
+	col.ColumnID = 2
+	if rec, err := ci.NextRecord(); rec != col || err != nil {
+		t.Fatalf("third record wrong: %v, err: %v", rec, err)
+	}
+	if rec, err := ci.NextRecord(); err != io.EOF {
+		t.Fatalf("should be EOF, but got %v, err: %v", rec, err)
+	}
+}
diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml
new file mode 100644
index 000000000..752c39177
--- /dev/null
+++ b/internal/clustertests/docker-compose.yml
@@ -0,0 +1,53 @@
+version: '2'
+services: 
+  pilosa1:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-withgo
+    image: ptest
+    ports:
+      - "33455:10101"
+    environment:
+      - PILOSA_CLUSTER_COORDINATOR=true
+      - PILOSA_GOSSIP_SEEDS=pilosa1:14000
+    networks:
+      - pilosanet
+    command:
+      - "/pilosa server --bind pilosa1:10101"
+  pilosa2:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-withgo
+    image: ptest
+    ports:
+      - "33456:10101"
+    environment:
+      - PILOSA_GOSSIP_SEEDS=pilosa1:14000
+    networks:
+      - pilosanet
+    command:
+      - "/pilosa server --bind pilosa2:10101"
+  pilosa3:
+    build:
+      context: .
+    image: ptest
+    ports:
+      - "33457:10101"
+    environment:
+      - PILOSA_GOSSIP_SEEDS=pilosa1:14000,pilosa2:14000
+    networks:
+      - pilosanet
+    command:
+      - "/pilosa server --bind pilosa3:10101"
+  client1:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-withgo
+    networks:
+      - pilosanet
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command:
+      - "go test -v github.com/pilosa/pilosa/internal/clustertests"
+networks: 
+  pilosanet:
diff --git a/internal/udproxy/udproxy.go b/internal/udproxy/udproxy.go
deleted file mode 100644
index a79c0121f..000000000
--- a/internal/udproxy/udproxy.go
+++ /dev/null
@@ -1,151 +0,0 @@
-package udproxy
-
-import (
-	"bytes"
-	"io"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"golang.org/x/sync/errgroup"
-)
-
-type Proxy struct {
-	upstreamAddr *net.UDPAddr
-	conn         *net.UDPConn
-
-	drop     bool
-	dropLock sync.Mutex
-
-	quit chan struct{}
-	eg   errgroup.Group
-	// map from client address to upstream connection. We must maintain a
-	// separate connection to upstream for each client connection so that we can
-	// differentiate data sent back from upstream.
-	upstreams map[*net.UDPAddr]*net.UDPConn
-	// TODO - need to track a per-connection timeout so that "upstreams" doesn't
-	// grow indefinitely.
-}
-
-func New(listenIP string, listenPort int, upstreamIP string, upstreamPort int) (*Proxy, error) {
-	uc, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP(listenIP), Port: listenPort})
-	if err != nil {
-		return nil, errors.Wrap(err, "listening")
-	}
-	p := &Proxy{
-		conn:         uc,
-		upstreamAddr: &net.UDPAddr{IP: net.ParseIP(upstreamIP), Port: upstreamPort},
-		quit:         make(chan struct{}),
-		upstreams:    make(map[*net.UDPAddr]*net.UDPConn),
-	}
-	if p.upstreamAddr.IP == nil {
-		return nil, errors.Errorf("unable to parse upstream ip '%s'", upstreamIP)
-	}
-	p.eg.Go(p.run)
-	return p, nil
-}
-
-func (p *Proxy) Drop() {
-	// sleep to attempt to ensure all traffic that was supposed to pass, did
-	// pass.
-	time.Sleep(time.Millisecond * 3)
-	p.dropLock.Lock()
-	p.drop = true
-	p.dropLock.Unlock()
-}
-
-func (p *Proxy) Undrop() {
-	// this sleep is a cheap attempt to ensure everything that was supposed to
-	// be dropped was.
-	time.Sleep(time.Millisecond * 3)
-	p.dropLock.Lock()
-	p.drop = false
-	p.dropLock.Unlock()
-}
-
-func (p *Proxy) dropping() bool {
-	p.dropLock.Lock()
-	d := p.drop
-	p.dropLock.Unlock()
-	return d
-}
-
-func (p *Proxy) run() error {
-	buf := make([]byte, 65507)
-	for {
-		select {
-		case <-p.quit:
-			return nil
-		default:
-		}
-		err := p.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
-		if err != nil {
-			return errors.Wrap(err, "setting read deadline (run)")
-		}
-		n, addr, err := p.conn.ReadFromUDP(buf)
-		if err, ok := err.(net.Error); ok && err.Timeout() {
-			continue
-		} else if err != nil {
-			return errors.Wrap(err, "reading from udp conn")
-		}
-		upConn := p.upstreams[addr]
-		if upConn == nil {
-			p.upstreams[addr], err = net.DialUDP("udp", &net.UDPAddr{}, p.upstreamAddr)
-			if err != nil {
-				return errors.Wrap(err, "creating new connection to upstream")
-			}
-			p.eg.Go(func() error {
-				return p.proxyBack(addr, p.upstreams[addr])
-			})
-			upConn = p.upstreams[addr]
-		}
-		if !p.dropping() {
-			_, err = io.Copy(upConn, bytes.NewBuffer(buf[:n]))
-			if err != nil {
-				return errors.Wrap(err, "writing to upstream conn")
-			}
-		}
-	}
-}
-
-func (p *Proxy) proxyBack(to *net.UDPAddr, from *net.UDPConn) error {
-	buf := make([]byte, 65507)
-	for {
-		select {
-		case <-p.quit:
-			return nil
-		default:
-		}
-		err := from.SetReadDeadline(time.Now().Add(time.Millisecond))
-		if err != nil {
-			return errors.Wrap(err, "setting read deadline (proxyBack)")
-		}
-		n, _, err := from.ReadFromUDP(buf)
-		if err, ok := err.(net.Error); ok && err.Timeout() {
-			continue
-		} else if err != nil {
-			return errors.Wrap(err, "reading from upstream")
-		}
-		if !p.dropping() {
-			_, err = io.Copy(addrWriter{c: p.conn, a: to}, bytes.NewBuffer(buf[:n]))
-			if err != nil {
-				return errors.Wrap(err, "writing back to client")
-			}
-		}
-	}
-}
-
-type addrWriter struct {
-	c *net.UDPConn
-	a *net.UDPAddr
-}
-
-func (a addrWriter) Write(b []byte) (n int, err error) {
-	return a.c.WriteTo(b, a.a)
-}
-
-func (p *Proxy) Close() error {
-	close(p.quit)
-	return p.eg.Wait()
-}
diff --git a/internal/udproxy/udproxy_test.go b/internal/udproxy/udproxy_test.go
deleted file mode 100644
index be746ea72..000000000
--- a/internal/udproxy/udproxy_test.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package udproxy_test
-
-import (
-	"net"
-	"testing"
-
-	"github.com/pilosa/pilosa/internal/udproxy"
-	"github.com/pkg/errors"
-	"golang.org/x/sync/errgroup"
-)
-
-func TestUDProxy(t *testing.T) {
-	p, err := udproxy.New("127.0.0.1", 12345, "127.0.0.1", 12346)
-	if err != nil {
-		t.Fatalf("creating proxy: %v", err)
-	}
-
-	uc, err := net.ListenUDP("udp", &net.UDPAddr{Port: 12346})
-	if err != nil {
-		t.Fatalf("listening udp upstream: %v", err)
-	}
-
-	resp := make([]byte, 8)
-	eg := errgroup.Group{}
-	eg.Go(func() error {
-		conn, err := net.DialUDP("udp", nil, &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 12345})
-		if err != nil {
-			t.Fatalf("connecting to proxy: %v", err)
-		}
-		_, err = conn.Write([]byte("hello!"))
-		if err != nil {
-			return errors.Wrap(err, "writing to proxy")
-		}
-		_, err = conn.Read(resp)
-		if err != nil {
-			return errors.Wrap(err, "reading from proxy")
-		}
-		p.Drop()
-		_, err = conn.Write([]byte("hello2"))
-		if err != nil {
-			return errors.Wrap(err, "writing to dropping proxy")
-		}
-		p.Undrop()
-		_, err = conn.Write([]byte("hello3"))
-		if err != nil {
-			return errors.Wrap(err, "writing to undropping proxy")
-		}
-		return nil
-	})
-
-	req := make([]byte, 10)
-	_, addr, err := uc.ReadFrom(req)
-	if err != nil {
-		t.Fatalf("upstream reading from proxy: %v", err)
-	}
-	if string(req[:6]) != "hello!" {
-		t.Fatalf("got unexpected request %s", req)
-	}
-	_, err = uc.WriteTo([]byte("goodbye"), addr)
-	if err != nil {
-		t.Fatalf("writing response: %v", err)
-	}
-
-	_, _, err = uc.ReadFrom(req)
-	if err != nil {
-		t.Fatalf("upstream reading from proxy: %v", err)
-	}
-	if string(req[:6]) != "hello3" {
-		t.Fatalf("got unexpected request %s", req)
-	}
-
-	eg.Wait()
-	if string(resp[:7]) != "goodbye" {
-		t.Fatalf("got unexpected response '%v", resp)
-	}
-
-	err = p.Close()
-	if err != nil {
-		t.Fatalf("err closing proxy: '%v'", err)
-	}
-}
-
-// TODO test dropping
diff --git a/server/server_test.go b/server/server_test.go
index f30c02774..d62605ce0 100644
--- a/server/server_test.go
+++ b/server/server_test.go
@@ -379,11 +379,7 @@ func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
 
 func TestClusteringNodesReplica1(t *testing.T) {
-	cluster := test.MustNewCluster(t, 3)
-	err := cluster.Start()
-	if err != nil {
-		t.Fatalf("starting cluster: %v", err)
-	}
+	cluster := test.MustRunCluster(t, 3)
 	defer cluster.Close()
 
 	var wait = true
@@ -696,12 +692,3 @@ func TestClusterQueriesAfterRestart(t *testing.T) {
 }
 
 // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address.
-
-func TestClusterPartitioning(t *testing.T) {
-	cluster := test.MustNewClusterWithProxy(t, 3)
-	err := cluster.Start()
-	if err != nil {
-		t.Fatalf("starting cluster with proxy: %v", err)
-	}
-
-}
diff --git a/test/pilosa.go b/test/pilosa.go
index 3efecfe73..3bc43d593 100644
--- a/test/pilosa.go
+++ b/test/pilosa.go
@@ -19,100 +19,28 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"math/rand"
 	gohttp "net/http"
 	"os"
 	"path"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/pilosa/pilosa"
 	"github.com/pilosa/pilosa/http"
-	"github.com/pilosa/pilosa/internal/udproxy"
 	"github.com/pilosa/pilosa/server"
 	"github.com/pkg/errors"
-
-	"github.com/Shopify/toxiproxy"
-	tox "github.com/Shopify/toxiproxy/client"
 )
 
-type portAllocator struct {
-	port *uint32
-}
-
-var ports *portAllocator
-
-// Next generates a new port and returns it
-func (n *portAllocator) Next() (nextPort uint32) {
-	return atomic.AddUint32(n.port, 1)
-}
-
-var proxy string
-
-func init() {
-	r := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
-	p := uint32(r.Uint32()%40000 + 20000)
-	ports = &portAllocator{
-		port: &p,
-	}
-
-	tport := strconv.Itoa(int(ports.Next()))
-	proxy = "localhost:" + tport
-	go toxiproxy.NewServer().Listen("localhost", tport)
-
-}
-
 ////////////////////////////////////////////////////////////////////////////////////
 // Command represents a test wrapper for server.Command.
 type Command struct {
 	*server.Command
 
-	proxies commandProxies
-
 	commandOptions []server.CommandOption
 }
 
-// Drop uses the proxies to drop all network traffic to/from this node.
-func (com Command) Drop(tb testing.TB) {
-	if com.proxies.http == nil {
-		tb.Fatal("can't drop traffic if cluster wasn't created with proxy support.")
-	}
-	err := com.proxies.http.Disable()
-	if err != nil {
-		tb.Fatalf("disabling http proxy: %v", err)
-	}
-	err = com.proxies.memberTCP.Disable()
-	if err != nil {
-		tb.Fatalf("disabling memberlist tcp proxy: %v", err)
-	}
-	com.proxies.memberUDP.Drop()
-}
-
-// Undrop starts forwarding traffic to/from this node after a previous drop request.
-func (com Command) Undrop(tb testing.TB) {
-	if com.proxies.http == nil {
-		tb.Fatal("can't drop traffic if cluster wasn't created with proxy support.")
-	}
-	err := com.proxies.http.Enable()
-	if err != nil {
-		tb.Fatalf("enabling http proxy: %v", err)
-	}
-	err = com.proxies.memberTCP.Enable()
-	if err != nil {
-		tb.Fatalf("enabling memberlist tcp proxy: %v", err)
-	}
-	com.proxies.memberUDP.Undrop()
-}
-
-type commandProxies struct {
-	http      *tox.Proxy
-	memberTCP *tox.Proxy
-	memberUDP *udproxy.Proxy
-}
-
 func OptAllowedOrigins(origins []string) server.CommandOption {
 	return func(m *server.Command) error {
 		m.Config.Handler.AllowedOrigins = origins
@@ -346,6 +274,7 @@ func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptio
 func (c Cluster) Start() error {
 	var gossipSeeds = make([]string, len(c))
 	for i, cc := range c {
+		cc.Config.Gossip.Port = "0"
 		cc.Config.Gossip.Seeds = gossipSeeds[:i]
 		if err := cc.Start(); err != nil {
 			return errors.Wrapf(err, "starting server %d", i)
@@ -367,17 +296,7 @@ func (c Cluster) Close() error {
 
 // MustNewCluster creates a new cluster
 func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
-	c, err := newCluster(size, false, opts...)
-	if err != nil {
-		tb.Fatalf("new cluster: %v", err)
-	}
-	return c
-}
-
-// MustNewClusterWithProxy returns a test cluster which has all connections
-// going through a proxy to allow for testing network partitions.
-func MustNewClusterWithProxy(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
-	c, err := newCluster(size, true, opts...)
+	c, err := newCluster(size, opts...)
 	if err != nil {
 		tb.Fatalf("new cluster: %v", err)
 	}
@@ -385,7 +304,7 @@ func MustNewClusterWithProxy(tb testing.TB, size int, opts ...[]server.CommandOp
 }
 
 // newCluster creates a new cluster
-func newCluster(size int, withproxy bool, opts ...[]server.CommandOption) (cluster Cluster, err error) {
+func newCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
 	if size == 0 {
 		return nil, errors.New("cluster must contain at least one node")
 	}
@@ -393,41 +312,14 @@ func newCluster(size int, withproxy bool, opts ...[]server.CommandOption) (clust
 		return nil, errors.New("Slice of CommandOptions must be of length 0, 1, or equal to the number of cluster nodes")
 	}
 
-	cluster = make(Cluster, size)
+	cluster := make(Cluster, size)
 	for i := 0; i < size; i++ {
 		var commandOpts []server.CommandOption
 		if len(opts) > 0 {
 			commandOpts = opts[i%len(opts)]
 		}
-		name := "node" + strconv.Itoa(i)
 		m := NewCommandNode(i == 0, commandOpts...)
-		m.Config.Bind = "localhost:" + strconv.Itoa(int(ports.Next()))
-		gossipBindPort := int(ports.Next())
-		m.Config.Gossip.Port = strconv.Itoa(gossipBindPort)
-
-		if withproxy {
-			tclient := tox.NewClient(proxy)
-
-			aport := strconv.Itoa(int(ports.Next()))
-			m.Config.Advertise = "localhost:" + aport
-			m.proxies.http, err = tclient.CreateProxy(name+aport, m.Config.Advertise, m.Config.Bind)
-			if err != nil {
-				return nil, errors.Wrap(err, "setting up toxiproxy")
-			}
-			gossipAdvertPort := int(ports.Next())
-			m.Config.Gossip.AdvertisePort = strconv.Itoa(gossipAdvertPort)
-			m.proxies.memberTCP, err = tclient.CreateProxy(name+"-gossip"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.AdvertisePort, "localhost:"+m.Config.Gossip.Port)
-			if err != nil {
-				return nil, errors.Wrap(err, "setting up toxiproxy for gossip")
-			}
-			m.proxies.memberUDP, err = udproxy.New("127.0.0.1", gossipAdvertPort, "127.0.0.1", gossipBindPort)
-			if err != nil {
-				return nil, errors.Wrap(err, "setting up proxy for udp gossip")
-			}
-
-		}
-
-		err = ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name), 0600)
+		err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte("node"+strconv.Itoa(i)), 0600)
 		if err != nil {
 			return nil, errors.Wrap(err, "writing node id")
 		}
@@ -438,8 +330,8 @@ func newCluster(size int, withproxy bool, opts ...[]server.CommandOption) (clust
 }
 
 // runCluster creates and starts a new cluster
-func runCluster(size int, withproxy bool, opts ...[]server.CommandOption) (Cluster, error) {
-	cluster, err := newCluster(size, withproxy, opts...)
+func runCluster(size int, opts ...[]server.CommandOption) (Cluster, error) {
+	cluster, err := newCluster(size, opts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "new cluster")
 	}
@@ -451,7 +343,7 @@ func runCluster(size int, withproxy bool, opts ...[]server.CommandOption) (Clust
 
 // MustRunCluster creates and starts a new cluster
 func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Cluster {
-	c, err := runCluster(size, false, opts...)
+	c, err := runCluster(size, opts...)
 	if err != nil {
 		tb.Fatalf("run cluster: %v", err)
 	}

From 02aee931a1c8d80743cdefd57506ef05cd42f62d Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Fri, 2 Nov 2018 16:03:50 -0500
Subject: [PATCH 07/20] few small fixes

---
 Dockerfile-withgo                     | 4 ++++
 internal/clustertests/cluster_test.go | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/Dockerfile-withgo b/Dockerfile-withgo
index 5cb11e47b..306d16f3e 100644
--- a/Dockerfile-withgo
+++ b/Dockerfile-withgo
@@ -1,3 +1,6 @@
+# This Dockerfile is used for cluster testing - it produces a much larger image
+# and includes all of Go as well as some utilities.
+
 FROM golang:1.11
 
 LABEL maintainer "dev@pilosa.com"
@@ -7,6 +10,7 @@ COPY . /go/src/github.com/pilosa/pilosa/
 RUN cd /go/src/github.com/pilosa/pilosa \
     && CGO_ENABLED=0 make install-dep install FLAGS="-a"
 
+# download pumba for fault injection
 RUN wget https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 -O /pumba
 RUN chmod +x /pumba
 
diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go
index e63b95d0b..10ffc638c 100644
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@@ -58,6 +58,7 @@ func TestLongPauses(t *testing.T) {
 	if err != nil {
 		t.Fatalf("waiting on pumba pause cmd: %v", err)
 	}
+	// TODO change the sleep to wait for status to return to NORMAL or timeout once we have Status.State support in go-pilosa
 	fmt.Println("done with pause, waiting for stability")
 	time.Sleep(time.Second * 400)
 	fmt.Println("done waiting")
@@ -79,6 +80,7 @@ func getPilosaClient(t *testing.T) *pilosa.Client {
 		time.Sleep(time.Millisecond * 40)
 	}
 	time.Sleep(time.Second * 2)
+	// TODO uncomment the following once we get the version of go-pilosa that has the State field on Status.
 	// start := time.Now()
 	// for i := 0; true; i++ {
 	// 	s, err := cli.Status()

From ec3982c0ff2ea1e0f20a8aba939ac2c631513c95 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Fri, 2 Nov 2018 16:10:39 -0500
Subject: [PATCH 08/20] pull out advertise URI changes

---
 ctl/server.go         |  1 -
 gossip/gossip.go      | 28 ++++------------------------
 server.go             | 16 ++++------------
 server/config.go      |  6 ------
 server/server.go      |  6 ------
 server/server_test.go |  1 +
 uri.go                | 17 ++++-------------
 7 files changed, 13 insertions(+), 62 deletions(-)

diff --git a/ctl/server.go b/ctl/server.go
index 90766c498..7f384ec7b 100644
--- a/ctl/server.go
+++ b/ctl/server.go
@@ -26,7 +26,6 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags := cmd.Flags()
 	flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.")
 	flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.")
-	flags.StringVarP(&srv.Config.Advertise, "advertise", "a", "", "Address to broadcast to other hosts and clients to be contacted on.")
 	flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.")
 	flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path")
 	flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging")
diff --git a/gossip/gossip.go b/gossip/gossip.go
index d03016792..789ed50b0 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -205,18 +205,8 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem
 	conf.Name = api.Node().ID
 	conf.BindAddr = api.Node().URI.Host
 	conf.BindPort = port
-	if cfg.AdvertisePort != "" {
-		port, err = strconv.Atoi(cfg.Port)
-		if err != nil {
-			return nil, fmt.Errorf("convert advertise port: %s", err)
-		}
-	}
 	conf.AdvertisePort = port
-	if cfg.AdvertiseHost != "" {
-		conf.AdvertiseAddr = cfg.AdvertiseHost
-	} else {
-		conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
-	}
+	conf.AdvertiseAddr = hostToIP(api.Node().URI.Host)
 	//
 	conf.TCPTimeout = time.Duration(cfg.StreamTimeout)
 	conf.SuspicionMult = cfg.SuspicionMult
@@ -457,20 +447,10 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) {
 
 // Config holds toml-friendly memberlist configuration.
 type Config struct {
-	// Host is the host gossip will bind to. If left blank it will be set to the
-	// host from Pilosa.
-	Host string `toml:"host"`
 	// Port indicates the port to which pilosa should bind for internal state sharing.
-	Port string `toml:"port"`
-	// AdvertiseHost is the hostname or IP other nodes should use to connect to
-	// this host. If left blank, the value for Host will be used. This is useful
-	// in some proxy and NAT scenarios.
-	AdvertiseHost string `toml:"advertise-host`
-	// AdvertisePort is the port other nodes will use to connect to this one.
-	// Behaves like AdvertiseHost.
-	AdvertisePort string   `toml:"advertise-port"`
-	Seeds         []string `toml:"seeds"`
-	Key           string   `toml:"key"`
+	Port  string   `toml:"port"`
+	Seeds []string `toml:"seeds"`
+	Key   string   `toml:"key"`
 	// StreamTimeout is the timeout for establishing a stream connection with
 	// a remote node for a full state sync, and for stream read and write
 	// operations. Maps to memberlist TCPTimeout.
diff --git a/server.go b/server.go
index 36190e315..a4cb8fc8b 100644
--- a/server.go
+++ b/server.go
@@ -62,7 +62,6 @@ type Server struct { // nolint: maligned
 
 	nodeID              string
 	uri                 URI
-	advertiseURI        URI
 	antiEntropyInterval time.Duration
 	metricInterval      time.Duration
 	diagnosticInterval  time.Duration
@@ -74,7 +73,7 @@ type Server struct { // nolint: maligned
 	dataDir       string
 }
 
-// TODO (2.0): have this return an interface for Holder instead of concrete object?
+// TODO: have this return an interface for Holder instead of concrete object?
 func (s *Server) Holder() *Holder {
 	return s.holder
 }
@@ -161,13 +160,6 @@ func OptServerInternalClient(c InternalClient) ServerOption {
 	}
 }
 
-func OptServerAdvertiseURI(u *URI) ServerOption {
-	return func(s *Server) error {
-		s.advertiseURI = *u
-		return nil
-	}
-}
-
 // DEPRECATED
 func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption {
 	return func(s *Server) error {
@@ -304,7 +296,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	// Set Cluster Node.
 	node := &Node{
 		ID:            s.nodeID,
-		URI:           s.advertiseURI,
+		URI:           s.uri,
 		IsCoordinator: s.cluster.Coordinator == s.nodeID,
 	}
 	s.cluster.Node = node
@@ -589,7 +581,7 @@ func (s *Server) SendSync(m Message) error {
 		node := node
 		s.logger.Printf("SendSync to: %s", node.URI)
 		// Don't forward the message to ourselves.
-		if s.advertiseURI == node.URI {
+		if s.uri == node.URI {
 			continue
 		}
 
@@ -684,7 +676,7 @@ func (s *Server) monitorDiagnostics() {
 
 	s.diagnostics.Logger = s.logger
 	s.diagnostics.SetVersion(Version)
-	s.diagnostics.Set("Host", s.advertiseURI.Host)
+	s.diagnostics.Set("Host", s.uri.Host)
 	s.diagnostics.Set("Cluster", strings.Join(s.cluster.nodeIDs(), ","))
 	s.diagnostics.Set("NumNodes", len(s.cluster.nodes))
 	s.diagnostics.Set("NumCPU", runtime.NumCPU())
diff --git a/server/config.go b/server/config.go
index 210a2e298..d255e4bb6 100644
--- a/server/config.go
+++ b/server/config.go
@@ -36,15 +36,9 @@ type Config struct {
 	// DataDir is the directory where Pilosa stores both indexed data and
 	// running state such as cluster topology information.
 	DataDir string `toml:"data-dir"`
-
 	// Bind is the host:port on which Pilosa will listen.
 	Bind string `toml:"bind"`
 
-	// Advertise is the host:port that this node will report as its address to
-	// others. If left blank (the default), this will be set to the bind address
-	// once it is listening.
-	Advertise string `toml:"advertise"`
-
 	// MaxWritesPerRequest limits the number of mutating commands that can be in
 	// a single request to the server. This includes Set, Clear,
 	// SetRowAttrs & SetColumnAttrs.
diff --git a/server/server.go b/server/server.go
index 8521abdec..010eb2f8a 100644
--- a/server/server.go
+++ b/server/server.go
@@ -248,11 +248,6 @@ func (m *Command) SetupServer() error {
 		uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port))
 	}
 
-	advertURI, err := pilosa.NewURIFromAddressWithDefault(m.Config.Advertise, uri)
-	if err != nil {
-		return errors.Wrapf(err, "processing avertise address '%s'", m.Config.Advertise)
-	}
-
 	c := http.GetHTTPClient(TLSConfig)
 
 	// Primary store configuration is handled automatically now.
@@ -281,7 +276,6 @@ func (m *Command) SetupServer() error {
 		pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
 		pilosa.OptServerStatsClient(statsClient),
 		pilosa.OptServerURI(uri),
-		pilosa.OptServerAdvertiseURI(advertURI),
 		pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
 		pilosa.OptServerPrimaryTranslateStoreFunc(http.NewTranslateStore),
 		pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
diff --git a/server/server_test.go b/server/server_test.go
index d62605ce0..aa202a54c 100644
--- a/server/server_test.go
+++ b/server/server_test.go
@@ -405,6 +405,7 @@ func TestClusteringNodesReplica1(t *testing.T) {
 	// Create new main with the same config.
 	config := cluster[2].Command.Config
 	config.Translation.MapSize = 100000
+	// config.Bind = cluster[2].API.Node().URI.HostPort()
 
 	// this isn't necessary, but makes the test run way faster
 	config.Gossip.Port = strconv.Itoa(int(cluster[2].Command.GossipTransport().URI.Port))
diff --git a/uri.go b/uri.go
index 7cf985f0b..231457691 100644
--- a/uri.go
+++ b/uri.go
@@ -82,10 +82,6 @@ func NewURIFromAddress(address string) (*URI, error) {
 	return parseAddress(address)
 }
 
-func NewURIFromAddressWithDefault(address string, base *URI) (*URI, error) {
-	return parseAddressWithDefault(address, base)
-}
-
 // setScheme sets the scheme of this URI.
 func (u *URI) setScheme(scheme string) error {
 	m := schemeRegexp.FindStringSubmatch(scheme)
@@ -158,20 +154,20 @@ func (u URI) Type() string {
 	return "URI"
 }
 
-func parseAddressWithDefault(address string, def *URI) (uri *URI, err error) {
+func parseAddress(address string) (uri *URI, err error) {
 	m := addressRegexp.FindStringSubmatch(address)
 	if m == nil {
 		return nil, errors.New("invalid address")
 	}
-	scheme := def.Scheme
+	scheme := "http"
 	if m[2] != "" {
 		scheme = m[2]
 	}
-	host := def.Host
+	host := "localhost"
 	if m[3] != "" {
 		host = m[3]
 	}
-	var port = int(def.Port)
+	var port = 10101
 	if m[5] != "" {
 		port, err = strconv.Atoi(m[5])
 		if err != nil {
@@ -189,11 +185,6 @@ func parseAddressWithDefault(address string, def *URI) (uri *URI, err error) {
 	return uri, nil
 }
 
-func parseAddress(address string) (uri *URI, err error) {
-	u, err := parseAddressWithDefault(address, defaultURI())
-	return u, err
-}
-
 // MarshalJSON marshals URI into a JSON-encoded byte slice.
 func (u *URI) MarshalJSON() ([]byte, error) {
 	var output struct {

From 6d821afed93d24b03e859759b6fc1d3c5e2159df Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Fri, 2 Nov 2018 16:13:03 -0500
Subject: [PATCH 09/20] add note on skipped cluster test

---
 internal/clustertests/cluster_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go
index 10ffc638c..23d27fcba 100644
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@@ -13,7 +13,7 @@ import (
 )
 
 func TestLongPauses(t *testing.T) {
-	t.Skip()
+	t.Skip() // TODO figure out how to only run in the docker-compose environment
 	cli := getPilosaClient(t)
 
 	idx := pilosa.NewIndex("testidx")

From deae8ce7c03336615a8df800b4dc97d3ad2d779e Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Fri, 9 Nov 2018 11:28:19 -0600
Subject: [PATCH 10/20] improvements to clustertests and fix cluster pause bug
 by state sharing

---
 Dockerfile-withgo                        |   2 +-
 Makefile                                 |  16 ++
 cluster.go                               |  51 ++++--
 cluster_internal_test.go                 |  16 +-
 encoding/proto/proto.go                  |   2 +
 internal/clustertests/cluster_test.go    | 112 +++++++------
 internal/clustertests/docker-compose.yml |   7 +-
 internal/private.pb.go                   | 190 ++++++++++++++---------
 internal/private.proto                   |   1 +
 server.go                                |   6 +-
 server/server.go                         |   2 +-
 11 files changed, 260 insertions(+), 145 deletions(-)

diff --git a/Dockerfile-withgo b/Dockerfile-withgo
index 306d16f3e..834634b13 100644
--- a/Dockerfile-withgo
+++ b/Dockerfile-withgo
@@ -23,4 +23,4 @@ EXPOSE 10101
 VOLUME /data
 
 ENTRYPOINT ["bash", "-c"]
-CMD ["pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["/pilosa", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
diff --git a/Makefile b/Makefile
index b403fde23..9cf4cffaf 100644
--- a/Makefile
+++ b/Makefile
@@ -68,6 +68,22 @@ release: check-clean
 	$(MAKE) release-build GOOS=linux GOARCH=386
 	$(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1
 
+# Run cluster integration tests using docker. Requires docker daemon to be
+# running. This will catch changes to internal/clustertests/*.go, but if you
+# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
+# pilosa image.
+clustertests:
+	cd internal/clustertests;\
+	docker-compose down;\
+	docker-compose up;
+
+
+# Like clustertests, but rebuilds all images.
+clustertests-build:
+	cd internal/clustertests;\
+	docker-compose down;\
+	docker-compose up --build;
+
 # Create prerelease builds
 prerelease: vendor
 	$(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\)
diff --git a/cluster.go b/cluster.go
index c3bc67b4d..4b1ab9089 100644
--- a/cluster.go
+++ b/cluster.go
@@ -65,10 +65,11 @@ type Node struct {
 	ID            string `json:"id"`
 	URI           URI    `json:"uri"`
 	IsCoordinator bool   `json:"isCoordinator"`
+	State         string `json:"state"`
 }
 
 func (n Node) String() string {
-	return fmt.Sprintf("Node: %s", n.ID)
+	return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID[:6])
 }
 
 // Nodes represents a list of nodes.
@@ -456,7 +457,19 @@ func (c *cluster) unprotectedSetState(state string) {
 	}
 }
 
+func (c *cluster) setMyNodeState(state string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.Node.State = state
+	for i, n := range c.nodes {
+		if n.ID == c.Node.ID {
+			c.nodes[i].State = state
+		}
+	}
+}
+
 func (c *cluster) setNodeState(state string) error { // nolint: unparam
+	c.setMyNodeState(state)
 	if c.isCoordinator() {
 		return c.receiveNodeState(c.Node.ID, state)
 	}
@@ -486,11 +499,23 @@ func (c *cluster) receiveNodeState(nodeID string, state string) error {
 	}
 
 	c.Topology.mu.Lock()
-	c.Topology.nodeStates[nodeID] = state
+	changed := false
+	if c.Topology.nodeStates[nodeID] != state {
+		changed = true
+		c.Topology.nodeStates[nodeID] = state
+		for i, n := range c.nodes {
+			if n.ID == nodeID {
+				c.nodes[i].State = state
+			}
+		}
+	}
 	c.Topology.mu.Unlock()
 	c.logger.Printf("received state %s (%s)", state, nodeID)
 
-	return c.unprotectedSetStateAndBroadcast(c.determineClusterState())
+	if changed {
+		return c.unprotectedSetStateAndBroadcast(c.determineClusterState())
+	}
+	return nil
 }
 
 // determineClusterState is unprotected.
@@ -932,7 +957,6 @@ func (c *cluster) waitForStarted() error {
 		<-c.joining
 		c.logger.Printf("joining has completed")
 	}
-
 	return nil
 }
 
@@ -1043,8 +1067,9 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error {
 		return nil
 	}
 	// Broadcast cluster status changes to the cluster.
-	c.logger.Printf("broadcasting ClusterStatus: %s", state)
-	return c.broadcaster.SendSync(c.unprotectedStatus()) // TODO fix c.Status
+	status := c.unprotectedStatus()
+	c.logger.Printf("broadcasting ClusterStatus: %s", status)
+	return c.broadcaster.SendSync(status) // TODO fix c.Status
 
 }
 
@@ -1625,7 +1650,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
 
 	switch e.Event {
 	case NodeJoin:
-		c.logger.Printf("received NodeJoin event: %v", e)
+		c.logger.Printf("nodeJoin of %s on %s", e.Node.URI, c.Node.URI)
 		// Ignore the event if this is not the coordinator.
 		if !c.isCoordinator() {
 			return nil
@@ -1660,6 +1685,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
 func (c *cluster) nodeJoin(node *Node) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	c.logger.Printf("NodeJoin event on coordinator, node: %s, id: %s", node.URI, node.ID)
 	if c.needTopologyAgreement() {
 		// A host that is not part of the topology can't be added to the STARTING cluster.
 		if !c.Topology.ContainsID(node.ID) {
@@ -1688,11 +1714,10 @@ func (c *cluster) nodeJoin(node *Node) error {
 
 		if c.haveTopologyAgreement() && c.allNodesReady() {
 			return c.unprotectedSetStateAndBroadcast(ClusterStateNormal)
-		} else {
-			// Send the status to the remote node. This lets the remote node
-			// know that it can proceed with opening its Holder.
-			return c.sendTo(node, c.unprotectedStatus())
 		}
+		// Send the status to the remote node. This lets the remote node
+		// know that it can proceed with opening its Holder.
+		return c.sendTo(node, c.unprotectedStatus())
 	}
 
 	// If the cluster already contains the node, just send it the cluster status.
@@ -1796,6 +1821,10 @@ func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
 
 	// Add all nodes from the coordinator.
 	for _, node := range officialNodes {
+		if node.ID == c.Node.ID && node.State != c.Node.State {
+			c.logger.Printf("mismatched state in mergeClusterStatus got %v have %v", node.State, c.Node.State)
+			go c.setNodeState(c.Node.State)
+		}
 		if err := c.addNode(node); err != nil {
 			return errors.Wrap(err, "adding node")
 		}
diff --git a/cluster_internal_test.go b/cluster_internal_test.go
index a2b529fba..50602d385 100644
--- a/cluster_internal_test.go
+++ b/cluster_internal_test.go
@@ -185,8 +185,8 @@ func TestFragSources(t *testing.T) {
 				"node0": {},
 				"node1": {},
 				"node2": {
-					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(2)},
+					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
+					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -197,11 +197,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{"node1", URI{"http", "host1", 10101}, false}, "i", "f", "standard", uint64(1)},
+					{&Node{ID: "node1", URI: URI{"http", "host1", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(1)},
 				},
 				"node1": {
-					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(2)},
+					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
+					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
 				},
 			},
 			err: "",
@@ -212,11 +212,11 @@ func TestFragSources(t *testing.T) {
 			idx:  idx,
 			expected: map[string][]*ResizeSource{
 				"node0": {
-					{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(0)},
-					{&Node{"node2", URI{"http", "host2", 10101}, false}, "i", "f", "standard", uint64(2)},
+					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(0)},
+					{&Node{ID: "node2", URI: URI{"http", "host2", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(2)},
 				},
 				"node1": {
-					{&Node{"node0", URI{"http", "host0", 10101}, false}, "i", "f", "standard", uint64(3)},
+					{&Node{ID: "node0", URI: URI{"http", "host0", 10101}, IsCoordinator: false}, "i", "f", "standard", uint64(3)},
 				},
 				"node2": {},
 			},
diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go
index e8de4ea1a..9826c7ef9 100644
--- a/encoding/proto/proto.go
+++ b/encoding/proto/proto.go
@@ -506,6 +506,7 @@ func encodeNode(n *pilosa.Node) *internal.Node {
 		ID:            n.ID,
 		URI:           encodeURI(n.URI),
 		IsCoordinator: n.IsCoordinator,
+		State:         n.State,
 	}
 }
 
@@ -761,6 +762,7 @@ func decodeNode(node *internal.Node, m *pilosa.Node) {
 	m.ID = node.ID
 	decodeURI(node.URI, &m.URI)
 	m.IsCoordinator = node.IsCoordinator
+	m.State = node.State
 }
 
 func decodeURI(i *internal.URI, m *pilosa.URI) {
diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go
index 23d27fcba..21c9e35de 100644
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@@ -12,63 +12,77 @@ import (
 	pi "github.com/pilosa/pilosa"
 )
 
-func TestLongPauses(t *testing.T) {
-	t.Skip() // TODO figure out how to only run in the docker-compose environment
+func TestClusterStuff(t *testing.T) {
+	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
+		t.Skip()
+	}
 	cli := getPilosaClient(t)
 
-	idx := pilosa.NewIndex("testidx")
-	err := cli.CreateIndex(idx)
-	if err != nil {
-		t.Fatalf("creating index: %v", err)
-	}
-	f := idx.Field("testf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
-	err = cli.CreateField(f)
-	if err != nil {
-		t.Fatalf("creating field: %v", err)
-	}
+	t.Run("long pause", func(t *testing.T) {
+		idx := pilosa.NewIndex("testidx")
+		err := cli.CreateIndex(idx)
+		if err != nil {
+			t.Fatalf("creating index: %v", err)
+		}
+		f := idx.Field("testf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
+		err = cli.CreateField(f)
+		if err != nil {
+			t.Fatalf("creating field: %v", err)
+		}
 
-	data := make([]pilosa.Column, 1000)
-	for i := range data {
-		data[i].RowID = 0
-		data[i].ColumnID = uint64((i/10)*pi.ShardWidth + i%10)
-	}
+		data := make([]pilosa.Column, 1000)
+		for i := range data {
+			data[i].RowID = 0
+			data[i].ColumnID = uint64((i/10)*pi.ShardWidth + i%10)
+		}
 
-	err = cli.ImportField(f, &colIterator{cols: data}, pilosa.OptImportBatchSize(1000))
-	if err != nil {
-		t.Fatalf("importing: %v", err)
-	}
+		err = cli.ImportField(f, &colIterator{cols: data}, pilosa.OptImportBatchSize(1000))
+		if err != nil {
+			t.Fatalf("importing: %v", err)
+		}
 
-	r, err := cli.Query(idx.Count(f.Row(0)))
-	if err != nil {
-		t.Fatalf("count querying: %v", err)
-	}
-	if r.Result().Count() != 1000 {
-		t.Fatalf("count after import is %d", r.Result().Count())
-	}
+		r, err := cli.Query(idx.Count(f.Row(0)))
+		if err != nil {
+			t.Fatalf("count querying: %v", err)
+		}
+		if r.Result().Count() != 1000 {
+			t.Fatalf("count after import is %d", r.Result().Count())
+		}
 
-	pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "30s")
-	pcmd.Stdout = os.Stdout
-	pcmd.Stderr = os.Stderr
-	fmt.Println("pausing pilosa3 for 30s")
-	err = pcmd.Start()
-	if err != nil {
-		t.Fatalf("starting pumba command: %v", err)
-	}
-	err = pcmd.Wait()
-	if err != nil {
-		t.Fatalf("waiting on pumba pause cmd: %v", err)
-	}
-	// TODO change the sleep to wait for status to return to NORMAL or timeout once we have Status.State support in go-pilosa
-	fmt.Println("done with pause, waiting for stability")
-	time.Sleep(time.Second * 400)
-	fmt.Println("done waiting")
+		pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s")
+		pcmd.Stdout = os.Stdout
+		pcmd.Stderr = os.Stderr
+		fmt.Println("pausing pilosa3 for 10s")
+		err = pcmd.Start()
+		if err != nil {
+			t.Fatalf("starting pumba command: %v", err)
+		}
+		err = pcmd.Wait()
+		if err != nil {
+			t.Fatalf("waiting on pumba pause cmd: %v", err)
+		}
+		// TODO change the sleep to wait for status to return to NORMAL or timeout once we have Status.State support in go-pilosa
+		fmt.Println("done with pause, waiting for stability")
+		time.Sleep(time.Second * 3)
+		fmt.Println("done waiting")
 
-	r, err = cli.Query(idx.Count(f.Row(0)))
+		r, err = cli.Query(idx.Count(f.Row(0)))
+		if err != nil {
+			t.Fatalf("count querying: %v", err)
+		} else if r.Result().Count() != 1000 {
+			t.Fatalf("count after import is %d", r.Result().Count())
+		}
+
+		fmt.Println("at the bottom")
+
+	})
+
+	down := exec.Command("/pumba", "stop", "clustertests_pilosa3_1", "clustertests_pilosa2_1", "clustertests_pilosa1_1")
+	down.Stdout = os.Stdout
+	down.Stderr = os.Stderr
+	err := down.Run()
 	if err != nil {
-		t.Fatalf("count querying: %v", err)
-	}
-	if r.Result().Count() != 1000 {
-		t.Fatalf("count after import is %d", r.Result().Count())
+		t.Logf("stopping Pilosa: %v", err)
 	}
 }
 
diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml
index 752c39177..7196ea0eb 100644
--- a/internal/clustertests/docker-compose.yml
+++ b/internal/clustertests/docker-compose.yml
@@ -41,13 +41,14 @@ services:
       - "/pilosa server --bind pilosa3:10101"
   client1:
     build:
-      context: ../..
-      dockerfile: Dockerfile-withgo
+      context: .
+    environment:
+      - ENABLE_PILOSA_CLUSTER_TESTS=1
     networks:
       - pilosanet
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     command:
-      - "go test -v github.com/pilosa/pilosa/internal/clustertests"
+      - "go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests"
 networks: 
   pilosanet:
diff --git a/internal/private.pb.go b/internal/private.pb.go
index be4719e2f..d5e5d9251 100644
--- a/internal/private.pb.go
+++ b/internal/private.pb.go
@@ -540,6 +540,7 @@ type Node struct {
 	ID            string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"`
 	URI           *URI   `protobuf:"bytes,2,opt,name=URI" json:"URI,omitempty"`
 	IsCoordinator bool   `protobuf:"varint,3,opt,name=IsCoordinator,proto3" json:"IsCoordinator,omitempty"`
+	State         string `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"`
 }
 
 func (m *Node) Reset()                    { *m = Node{} }
@@ -568,6 +569,13 @@ func (m *Node) GetIsCoordinator() bool {
 	return false
 }
 
+func (m *Node) GetState() string {
+	if m != nil {
+		return m.State
+	}
+	return ""
+}
+
 type NodeStateMessage struct {
 	NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"`
 	State  string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
@@ -1749,6 +1757,12 @@ func (m *Node) MarshalTo(dAtA []byte) (int, error) {
 		}
 		i++
 	}
+	if len(m.State) > 0 {
+		dAtA[i] = 0x22
+		i++
+		i = encodeVarintPrivate(dAtA, i, uint64(len(m.State)))
+		i += copy(dAtA[i:], m.State)
+	}
 	return i, nil
 }
 
@@ -2675,6 +2689,10 @@ func (m *Node) Size() (n int) {
 	if m.IsCoordinator {
 		n += 2
 	}
+	l = len(m.State)
+	if l > 0 {
+		n += 1 + l + sovPrivate(uint64(l))
+	}
 	return n
 }
 
@@ -5227,6 +5245,35 @@ func (m *Node) Unmarshal(dAtA []byte) error {
 				}
 			}
 			m.IsCoordinator = bool(v != 0)
+		case 4:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field State", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPrivate
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.State = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipPrivate(dAtA[iNdEx:])
@@ -7399,75 +7446,76 @@ var (
 func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
 
 var fileDescriptorPrivate = []byte{
-	// 1113 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45,
-	0x18, 0x66, 0x0f, 0x76, 0xec, 0xdf, 0x75, 0x9a, 0x6c, 0x69, 0xd9, 0x02, 0x0a, 0x61, 0x54, 0xd1,
-	0x50, 0x89, 0x50, 0xb5, 0x37, 0x9c, 0x2a, 0x95, 0xc4, 0xa1, 0x2c, 0x25, 0xa5, 0xcc, 0xa6, 0xb9,
-	0xeb, 0xc5, 0xc4, 0x1e, 0x35, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x12, 0xf7, 0x82, 0x5b, 0x90,
-	0x78, 0x01, 0xc4, 0x93, 0xf0, 0x08, 0x5c, 0xf2, 0x08, 0x28, 0xbc, 0x08, 0x9a, 0x7f, 0x66, 0x76,
-	0x37, 0x8e, 0x43, 0xa2, 0xc0, 0xdd, 0xfc, 0xdf, 0x7f, 0x3e, 0xae, 0x0d, 0xfd, 0x49, 0x9e, 0x1c,
-	0x32, 0xc9, 0xd7, 0x27, 0xb9, 0x90, 0x22, 0xe8, 0x24, 0x99, 0xe4, 0x79, 0xc6, 0x52, 0xf2, 0x04,
-	0xba, 0x51, 0x36, 0xe2, 0xc7, 0xdb, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x29, 0x9f, 0x16, 0xa1, 0xb7,
-	0xea, 0xac, 0x75, 0x28, 0xbe, 0x83, 0x0f, 0x60, 0x71, 0x27, 0x67, 0xc3, 0x83, 0xad, 0xe3, 0xa4,
-	0x90, 0x3c, 0x1b, 0xf2, 0xd0, 0x47, 0xee, 0x0c, 0x4a, 0x7e, 0x77, 0xe0, 0xda, 0x57, 0x09, 0x4f,
-	0x47, 0xdf, 0x4d, 0x64, 0x22, 0xb2, 0x22, 0x78, 0x17, 0xba, 0x9b, 0x6c, 0xb8, 0xcf, 0x77, 0xa6,
-	0x13, 0x8e, 0x16, 0xbb, 0xb4, 0x06, 0x2a, 0x6e, 0x9c, 0xbc, 0xd6, 0x16, 0xfb, 0xb4, 0x06, 0x82,
-	0x55, 0xe8, 0xed, 0x24, 0x63, 0xfe, 0x7d, 0xc9, 0x32, 0x59, 0x8e, 0xc3, 0x16, 0x6a, 0x37, 0x21,
-	0x15, 0x2a, 0x1a, 0xee, 0x20, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0xdb, 0x49, 0x16, 0x76, 0x57, 0x9d,
-	0x35, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x38, 0x04, 0x83, 0xb0, 0xe3, 0x2a, 0xc5, 0x5e, 0x9d, 0x22,
-	0x21, 0xb0, 0x18, 0x8d, 0x27, 0x22, 0x97, 0x94, 0x17, 0x13, 0x91, 0x15, 0x68, 0x69, 0x2b, 0xcf,
-	0x43, 0x07, 0x8d, 0xab, 0x27, 0xf9, 0x11, 0x96, 0x36, 0x52, 0x31, 0x3c, 0x18, 0x30, 0xc9, 0x28,
-	0xff, 0xa1, 0xe4, 0x85, 0x0c, 0xde, 0x84, 0x16, 0xd6, 0xce, 0xc8, 0x69, 0x42, 0xa1, 0x58, 0x87,
-	0xd0, 0xd5, 0x28, 0x12, 0x0a, 0x45, 0x7d, 0xac, 0x84, 0x4f, 0x35, 0xa1, 0xd0, 0x78, 0x9f, 0xe5,
-	0x23, 0xac, 0x80, 0x4f, 0x35, 0xa1, 0x62, 0xdc, 0x4d, 0xf8, 0x91, 0x49, 0x1b, 0xdf, 0x24, 0x82,
-	0xe5, 0x86, 0x7f, 0x13, 0xe6, 0x2d, 0x68, 0x53, 0x71, 0x14, 0x0d, 0x8a, 0xd0, 0x59, 0xf5, 0xd6,
-	0x7c, 0x6a, 0x28, 0x2c, 0xae, 0x48, 0xcb, 0x71, 0xa6, 0x58, 0x2e, 0xb2, 0x6a, 0x80, 0xdc, 0x86,
-	0x16, 0x56, 0x5a, 0x65, 0x59, 0xeb, 0xaa, 0x27, 0xf9, 0xc9, 0x81, 0xee, 0x36, 0x3b, 0xc6, 0x30,
-	0x8a, 0xe0, 0x11, 0x74, 0x62, 0xc9, 0xb2, 0x91, 0x0a, 0x50, 0x09, 0xf5, 0x1e, 0xbc, 0xbf, 0x6e,
-	0x07, 0x67, 0xbd, 0x12, 0x5b, 0xb7, 0x32, 0x5b, 0x99, 0xcc, 0xa7, 0xb4, 0x52, 0x79, 0xfb, 0x73,
-	0xe8, 0x9f, 0x62, 0x29, 0x7f, 0x07, 0x7c, 0x6a, 0xab, 0x7a, 0xc0, 0xa7, 0x2a, 0xff, 0x43, 0x96,
-	0x96, 0x1c, 0x6b, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x5d, 0x08, 0x36, 0x73, 0xce,
-	0x24, 0x47, 0x27, 0xdb, 0xbc, 0x28, 0xd8, 0x2b, 0x7e, 0x7e, 0xc5, 0x75, 0x15, 0xdd, 0x66, 0x15,
-	0xab, 0x3e, 0x78, 0x8d, 0x3e, 0x90, 0x7b, 0x10, 0x0c, 0x78, 0xca, 0x25, 0x37, 0x53, 0xff, 0x2f,
-	0x76, 0x49, 0x6c, 0x63, 0xb8, 0x58, 0x36, 0xb8, 0x0b, 0xbe, 0x5a, 0x21, 0x0c, 0xa1, 0xf7, 0xe0,
-	0x46, 0x5d, 0xa7, 0x6a, 0xbb, 0x28, 0x0a, 0x90, 0xd4, 0x1a, 0xc5, 0x78, 0x2e, 0x4c, 0x6c, 0xce,
-	0x28, 0xdd, 0x33, 0xae, 0x3c, 0x74, 0x75, 0xab, 0x76, 0xd5, 0x5c, 0x3f, 0xe3, 0xed, 0xb1, 0x4d,
-	0xf7, 0xaa, 0xde, 0xc8, 0x10, 0xde, 0xd1, 0x16, 0xbe, 0x3c, 0x64, 0x49, 0xca, 0xf6, 0xd2, 0x4b,
-	0x76, 0x64, 0x4e, 0xe0, 0x21, 0x2c, 0xa0, 0x6e, 0x34, 0x30, 0x5b, 0x60, 0x49, 0xf2, 0xd2, 0xc8,
-	0xab, 0xd1, 0x7f, 0xc6, 0xc6, 0xdc, 0x58, 0xc3, 0x77, 0x95, 0xaf, 0x7b, 0x71, 0xbe, 0xca, 0xb1,
-	0x5a, 0x17, 0x75, 0xc2, 0x3c, 0xe5, 0x18, 0x09, 0xf2, 0x10, 0xda, 0xf1, 0x70, 0x9f, 0x8f, 0x59,
-	0xf0, 0x21, 0x2c, 0x60, 0x84, 0xbc, 0x30, 0x13, 0x7d, 0x7d, 0xa6, 0x53, 0xd4, 0xf2, 0xc9, 0xc0,
-	0x64, 0x36, 0x37, 0xa6, 0xbb, 0xd0, 0x46, 0xef, 0x45, 0xe8, 0xcf, 0x9a, 0x41, 0x9c, 0x1a, 0x36,
-	0xd9, 0x02, 0xef, 0x05, 0x8d, 0xd4, 0xa6, 0x62, 0x04, 0xd6, 0x8a, 0xa1, 0x94, 0xed, 0xaf, 0x45,
-	0x21, 0x4d, 0x9d, 0xf0, 0xad, 0xb0, 0xe7, 0x22, 0x97, 0x58, 0xa3, 0x3e, 0xc5, 0x37, 0x79, 0x09,
-	0xfe, 0x33, 0x31, 0xe2, 0xc1, 0x22, 0xb8, 0xd1, 0xc0, 0xd8, 0x70, 0xa3, 0x41, 0xf0, 0x1e, 0x9a,
-	0x37, 0xa5, 0xe9, 0xd7, 0x41, 0xbc, 0xa0, 0x11, 0x45, 0xc7, 0x77, 0xa0, 0x1f, 0x15, 0x9b, 0x42,
-	0xe4, 0xa3, 0x24, 0x63, 0x52, 0xe4, 0xe6, 0xb6, 0x9f, 0x06, 0xc9, 0x63, 0x58, 0x52, 0xe6, 0x63,
-	0xc9, 0x24, 0xb7, 0x9d, 0xbd, 0x05, 0x6d, 0x85, 0x55, 0xee, 0x0c, 0x85, 0xdb, 0xa6, 0xe4, 0x6c,
-	0x6f, 0x91, 0x20, 0xdf, 0x6a, 0x0b, 0x5b, 0x87, 0x3c, 0x93, 0x8d, 0xd9, 0x40, 0x1a, 0x0d, 0xf4,
-	0xa9, 0x26, 0x02, 0xa2, 0x53, 0x31, 0x31, 0x2f, 0xd6, 0x31, 0x2b, 0x94, 0x22, 0x8f, 0xfc, 0xe2,
-	0x00, 0xd8, 0x80, 0xca, 0xa2, 0x52, 0x71, 0xce, 0x57, 0x09, 0xd6, 0x6c, 0x8f, 0xcd, 0x5e, 0x2c,
-	0xd5, 0x52, 0x1a, 0xa7, 0x76, 0x06, 0x3e, 0xae, 0x67, 0x40, 0x37, 0xef, 0xe6, 0xcc, 0x0c, 0x68,
-	0xaf, 0xf5, 0x24, 0x3c, 0x87, 0x5e, 0x03, 0x9f, 0x3b, 0x0f, 0x1f, 0x55, 0xf3, 0xe0, 0xce, 0x9a,
-	0x44, 0xdc, 0x98, 0xb4, 0x53, 0xf1, 0x14, 0x7a, 0x0d, 0x78, 0xae, 0xc5, 0x35, 0xb8, 0x7e, 0x7a,
-	0xe3, 0xec, 0x25, 0x9f, 0x85, 0x49, 0x02, 0xfd, 0xcd, 0xb4, 0x2c, 0x24, 0xcf, 0x8d, 0x39, 0x75,
-	0xfe, 0x35, 0x50, 0x35, 0xaf, 0x06, 0xe6, 0xf7, 0x2f, 0xb8, 0x03, 0x2d, 0x55, 0x46, 0xbd, 0x38,
-	0x67, 0x6b, 0xac, 0x99, 0x64, 0x17, 0x3a, 0x1b, 0x71, 0xf4, 0x24, 0x17, 0xe5, 0x64, 0x6e, 0xd0,
-	0xf6, 0xab, 0xec, 0x9e, 0xfd, 0x2a, 0x7b, 0x67, 0xbe, 0xca, 0x7e, 0xf5, 0x55, 0x26, 0x31, 0x2c,
-	0xeb, 0xa3, 0xa8, 0xf6, 0xf5, 0x2a, 0xa7, 0xc5, 0x7e, 0x32, 0xbd, 0xc6, 0x27, 0x33, 0x86, 0x65,
-	0x7d, 0xb9, 0xfe, 0x4f, 0xa3, 0xbf, 0xb9, 0xb0, 0x4c, 0x79, 0x91, 0xbc, 0xe6, 0x51, 0x56, 0xc8,
-	0xbc, 0x1c, 0xaa, 0xeb, 0xa3, 0xf4, 0xbf, 0x11, 0x7b, 0xa6, 0xda, 0x1e, 0xd5, 0xc4, 0x65, 0x26,
-	0x3d, 0xb8, 0x0f, 0xbd, 0xd9, 0xed, 0x3c, 0x2b, 0xda, 0x14, 0x09, 0xee, 0xc3, 0x42, 0x2c, 0xca,
-	0x7c, 0x58, 0x8d, 0x6f, 0xe3, 0x22, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xc6, 0x6a, 0xb4, 0x2e,
-	0x58, 0x8d, 0x47, 0x33, 0xa3, 0x14, 0xb6, 0x51, 0xe1, 0xad, 0x5a, 0xe1, 0x14, 0x9b, 0x9e, 0x96,
-	0x26, 0x3f, 0x3b, 0x70, 0xad, 0x19, 0xc2, 0xa5, 0x16, 0xb7, 0xea, 0x88, 0x3b, 0xb7, 0x23, 0xde,
-	0xbc, 0x8e, 0xf8, 0x75, 0x47, 0xea, 0xaf, 0x7f, 0xab, 0xf1, 0xf5, 0x27, 0x07, 0x70, 0xfb, 0x4c,
-	0x9b, 0x36, 0xc5, 0x78, 0xa2, 0xe6, 0xe1, 0x3f, 0xb4, 0x4b, 0x9d, 0xb4, 0x3c, 0x37, 0x8d, 0xea,
-	0x52, 0x4d, 0x90, 0x4f, 0xe1, 0x66, 0xcc, 0x65, 0xa3, 0x49, 0x76, 0xda, 0x56, 0xc1, 0x7b, 0xc6,
-	0x8f, 0xce, 0x49, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xbe, 0x98, 0x8c, 0x98, 0xe4, 0x57, 0xd2, 0xde,
-	0x80, 0xce, 0x8e, 0x98, 0x88, 0x54, 0xbc, 0x9a, 0x5e, 0xb0, 0xf5, 0x21, 0x2c, 0xe8, 0xfb, 0xad,
-	0xcf, 0x48, 0x97, 0x5a, 0x92, 0xdc, 0x50, 0x03, 0x3d, 0x64, 0xe9, 0xb0, 0x4c, 0x55, 0x18, 0xea,
-	0x97, 0x61, 0xb1, 0xb1, 0xf4, 0xc7, 0xc9, 0x8a, 0xf3, 0xe7, 0xc9, 0x8a, 0xf3, 0xd7, 0xc9, 0x8a,
-	0xf3, 0xeb, 0xdf, 0x2b, 0x6f, 0xec, 0xb5, 0xf1, 0x9f, 0xc3, 0xc3, 0x7f, 0x02, 0x00, 0x00, 0xff,
-	0xff, 0xba, 0x1b, 0x62, 0x68, 0x4a, 0x0c, 0x00, 0x00,
+	// 1121 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0xc5,
+	0x17, 0xff, 0xef, 0x87, 0x1d, 0xfb, 0xb8, 0x4e, 0x93, 0xed, 0xbf, 0x61, 0x0b, 0x28, 0x84, 0x51,
+	0x45, 0x43, 0x25, 0x42, 0xd5, 0xde, 0xf0, 0x55, 0xa9, 0x24, 0x0e, 0x65, 0x29, 0x09, 0x65, 0x9c,
+	0xe4, 0x8e, 0x8b, 0x89, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0x98, 0xdd, 0xd9, 0x24, 0xee, 0x05, 0xb7,
+	0x20, 0xf1, 0x02, 0x88, 0x27, 0xe1, 0x11, 0xb8, 0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0x67, 0x66,
+	0x76, 0x37, 0x8e, 0x83, 0xa3, 0xc0, 0xdd, 0x9c, 0xdf, 0x99, 0xf9, 0x9d, 0xef, 0xb3, 0x36, 0x74,
+	0xc7, 0x59, 0x7c, 0xc2, 0x24, 0xdf, 0x18, 0x67, 0x42, 0x8a, 0xa0, 0x15, 0xa7, 0x92, 0x67, 0x29,
+	0x4b, 0xc8, 0x73, 0x68, 0x47, 0xe9, 0x90, 0x9f, 0xed, 0x70, 0xc9, 0x82, 0x00, 0xfc, 0x17, 0x7c,
+	0x92, 0x87, 0xde, 0x9a, 0xb3, 0xde, 0xa2, 0x78, 0x0e, 0xde, 0x83, 0xc5, 0xbd, 0x8c, 0x0d, 0x8e,
+	0xb7, 0xcf, 0xe2, 0x5c, 0xf2, 0x74, 0xc0, 0x43, 0x1f, 0xb5, 0x53, 0x28, 0xf9, 0xcd, 0x81, 0x5b,
+	0x5f, 0xc4, 0x3c, 0x19, 0x7e, 0x33, 0x96, 0xb1, 0x48, 0xf3, 0xe0, 0x6d, 0x68, 0x6f, 0xb1, 0xc1,
+	0x11, 0xdf, 0x9b, 0x8c, 0x39, 0x32, 0xb6, 0x69, 0x05, 0x94, 0xda, 0x7e, 0xfc, 0x5a, 0x33, 0x76,
+	0x69, 0x05, 0x04, 0x6b, 0xd0, 0xd9, 0x8b, 0x47, 0xfc, 0xdb, 0x82, 0xa5, 0xb2, 0x18, 0x85, 0x0d,
+	0x7c, 0x5d, 0x87, 0x94, 0xab, 0x48, 0xdc, 0x42, 0x15, 0x9e, 0x83, 0x25, 0xf0, 0x76, 0xe2, 0x34,
+	0x6c, 0xaf, 0x39, 0xeb, 0x1e, 0x55, 0x47, 0x44, 0xd8, 0x59, 0x08, 0x06, 0x61, 0x67, 0x65, 0x88,
+	0x9d, 0x2a, 0x44, 0x42, 0x60, 0x31, 0x1a, 0x8d, 0x45, 0x26, 0x29, 0xcf, 0xc7, 0x22, 0xcd, 0x91,
+	0x69, 0x3b, 0xcb, 0x42, 0x07, 0xc9, 0xd5, 0x91, 0xfc, 0x00, 0x4b, 0x9b, 0x89, 0x18, 0x1c, 0xf7,
+	0x98, 0x64, 0x94, 0x7f, 0x5f, 0xf0, 0x5c, 0x06, 0xff, 0x87, 0x06, 0xe6, 0xce, 0xdc, 0xd3, 0x82,
+	0x42, 0x31, 0x0f, 0xa1, 0xab, 0x51, 0x14, 0x14, 0x8a, 0xef, 0x31, 0x13, 0x3e, 0xd5, 0x82, 0x42,
+	0xfb, 0x47, 0x2c, 0x1b, 0x62, 0x06, 0x7c, 0xaa, 0x05, 0xe5, 0xe3, 0x41, 0xcc, 0x4f, 0x4d, 0xd8,
+	0x78, 0x26, 0x11, 0x2c, 0xd7, 0xec, 0x1b, 0x37, 0x57, 0xa0, 0x49, 0xc5, 0x69, 0xd4, 0xcb, 0x43,
+	0x67, 0xcd, 0x5b, 0xf7, 0xa9, 0x91, 0x30, 0xb9, 0x22, 0x29, 0x46, 0xa9, 0x52, 0xb9, 0xa8, 0xaa,
+	0x00, 0x72, 0x0f, 0x1a, 0x98, 0x69, 0x15, 0x65, 0xf5, 0x56, 0x1d, 0xc9, 0x8f, 0x0e, 0xb4, 0x77,
+	0xd8, 0x19, 0xba, 0x91, 0x07, 0x4f, 0xa1, 0xd5, 0x97, 0x2c, 0x1d, 0x2a, 0x07, 0xd5, 0xa5, 0xce,
+	0xe3, 0x77, 0x37, 0x6c, 0xe3, 0x6c, 0x94, 0xd7, 0x36, 0xec, 0x9d, 0xed, 0x54, 0x66, 0x13, 0x5a,
+	0x3e, 0x79, 0xf3, 0x53, 0xe8, 0x5e, 0x50, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27,
+	0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c, 0x73, 0xe5, 0x53, 0x2d, 0x7c, 0xe2, 0x7e, 0xe4, 0x90, 0x03,
+	0x08, 0xb6, 0x32, 0xce, 0x24, 0x47, 0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x75, 0xc6, 0x75,
+	0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a, 0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25,
+	0x37, 0x5d, 0xff, 0x0f, 0xbc, 0xa4, 0x6f, 0x7d, 0x98, 0x7f, 0x37, 0x78, 0x00, 0xbe, 0x1a, 0x21,
+	0x74, 0xa1, 0xf3, 0xf8, 0x4e, 0x95, 0xa7, 0x72, 0xba, 0x28, 0x5e, 0x20, 0x89, 0x25, 0x45, 0x7f,
+	0xe6, 0x06, 0x36, 0xa3, 0x95, 0x1e, 0x1a, 0x53, 0x1e, 0x9a, 0x5a, 0xa9, 0x4c, 0xd5, 0xc7, 0xcf,
+	0x58, 0x7b, 0x66, 0xc3, 0xbd, 0xa9, 0x35, 0x32, 0x80, 0xb7, 0x34, 0xc3, 0xe7, 0x27, 0x2c, 0x4e,
+	0xd8, 0x61, 0x72, 0xcd, 0x8a, 0xcc, 0x70, 0x3c, 0x84, 0x05, 0x7c, 0x1b, 0xf5, 0xcc, 0x14, 0x58,
+	0x91, 0x7c, 0x67, 0xee, 0xab, 0xd6, 0xdf, 0x65, 0x23, 0x6e, 0xd8, 0xf0, 0x5c, 0xc6, 0xeb, 0xce,
+	0x8f, 0x57, 0x19, 0x56, 0xe3, 0xa2, 0x56, 0x98, 0xa7, 0x0c, 0xa3, 0x40, 0x9e, 0x40, 0xb3, 0x3f,
+	0x38, 0xe2, 0x23, 0x16, 0xbc, 0x0f, 0x0b, 0xe8, 0x21, 0xcf, 0x4d, 0x47, 0xdf, 0x9e, 0xaa, 0x14,
+	0xb5, 0x7a, 0xd2, 0x33, 0x91, 0xcd, 0xf4, 0xe9, 0x01, 0x34, 0xd1, 0x7a, 0x1e, 0xfa, 0xd3, 0x34,
+	0x88, 0x53, 0xa3, 0x26, 0xdb, 0xe0, 0xed, 0xd3, 0x48, 0x4d, 0x2a, 0x7a, 0x60, 0x59, 0x8c, 0xa4,
+	0xb8, 0xbf, 0x14, 0xb9, 0x34, 0x79, 0xc2, 0xb3, 0xc2, 0x5e, 0x8a, 0x4c, 0x62, 0x8e, 0xba, 0x14,
+	0xcf, 0x24, 0x07, 0x7f, 0x57, 0x0c, 0x79, 0xb0, 0x08, 0x6e, 0xd4, 0x33, 0x1c, 0x6e, 0xd4, 0x0b,
+	0xde, 0x41, 0x7a, 0x93, 0x9a, 0x6e, 0xe5, 0xc4, 0x3e, 0x8d, 0x28, 0x1a, 0xbe, 0x0f, 0xdd, 0x28,
+	0xdf, 0x12, 0x22, 0x1b, 0xc6, 0x29, 0x93, 0x22, 0x33, 0xbb, 0xfd, 0x22, 0x88, 0x13, 0x24, 0x99,
+	0xd4, 0x9b, 0xb8, 0x4d, 0xb5, 0x40, 0x9e, 0xc1, 0x92, 0x32, 0x8a, 0x82, 0xad, 0xf7, 0x0a, 0x34,
+	0x15, 0x56, 0x3a, 0x61, 0xa4, 0x8a, 0xc1, 0xad, 0x33, 0x7c, 0xad, 0x19, 0xb6, 0x4f, 0x78, 0x2a,
+	0x6b, 0x1d, 0x83, 0x32, 0x12, 0x74, 0xa9, 0x16, 0x02, 0xa2, 0x03, 0x34, 0x91, 0x2c, 0x56, 0x91,
+	0x28, 0x94, 0xa2, 0x8e, 0xfc, 0xec, 0x00, 0x58, 0x87, 0x8a, 0xbc, 0x7c, 0xe2, 0x5c, 0xfd, 0x24,
+	0x58, 0xb7, 0x95, 0x37, 0xd3, 0xb2, 0x54, 0xdd, 0xd2, 0x38, 0xb5, 0x9d, 0xf1, 0x61, 0xd5, 0x19,
+	0xba, 0xa4, 0x77, 0xa7, 0x3a, 0x43, 0x5b, 0xad, 0xfa, 0xe3, 0x25, 0x74, 0x6a, 0xf8, 0xcc, 0x2e,
+	0xf9, 0xa0, 0xec, 0x12, 0x77, 0x9a, 0x12, 0x71, 0x43, 0x69, 0x7b, 0xe5, 0x05, 0x74, 0x6a, 0xf0,
+	0x4c, 0xc6, 0x75, 0xb8, 0x7d, 0x71, 0x0e, 0xed, 0x7e, 0x9f, 0x86, 0x49, 0x0c, 0xdd, 0xad, 0xa4,
+	0xc8, 0x25, 0xcf, 0x0c, 0x9d, 0xfa, 0x28, 0x68, 0xa0, 0x2c, 0x5e, 0x05, 0xcc, 0xae, 0x5f, 0x70,
+	0x1f, 0x1a, 0x2a, 0x8d, 0x7a, 0x9c, 0x2e, 0xe7, 0x58, 0x2b, 0xc9, 0x01, 0xb4, 0x36, 0xfb, 0xd1,
+	0xf3, 0x4c, 0x14, 0xe3, 0x99, 0x4e, 0xdb, 0x6f, 0xb5, 0x7b, 0xf9, 0x5b, 0xed, 0x5d, 0xfa, 0x56,
+	0xfb, 0xe5, 0xb7, 0x9a, 0xf4, 0x61, 0x59, 0xaf, 0x4a, 0x35, 0xc5, 0x37, 0x59, 0x38, 0xf6, 0x43,
+	0xea, 0xd5, 0x3e, 0xa4, 0x7d, 0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x24, 0xfd, 0xd5, 0x85, 0x65, 0xca,
+	0xf3, 0xf8, 0x35, 0x8f, 0xd2, 0x5c, 0x66, 0xc5, 0x40, 0xed, 0x24, 0xf5, 0xfe, 0x2b, 0x71, 0x68,
+	0xb2, 0xed, 0x51, 0x2d, 0x5c, 0xa7, 0xd3, 0x83, 0x47, 0xd0, 0x99, 0x9e, 0xd9, 0xcb, 0x57, 0xeb,
+	0x57, 0x82, 0x47, 0xb0, 0xd0, 0x17, 0x45, 0x36, 0x28, 0xdb, 0xb7, 0xb6, 0x27, 0xb5, 0x67, 0x5a,
+	0x4d, 0xed, 0xb5, 0xda, 0x68, 0x34, 0xe6, 0x8c, 0xc6, 0xd3, 0xa9, 0x56, 0x0a, 0x9b, 0xf8, 0xe0,
+	0x8d, 0xea, 0xc1, 0x05, 0x35, 0xbd, 0x78, 0x9b, 0xfc, 0xe4, 0xc0, 0xad, 0xba, 0x0b, 0xd7, 0x1a,
+	0xdc, 0xb2, 0x22, 0xee, 0xcc, 0x8a, 0x78, 0xb3, 0x2a, 0xe2, 0x57, 0x15, 0xa9, 0x7e, 0x13, 0x34,
+	0x6a, 0xbf, 0x09, 0xc8, 0x31, 0xdc, 0xbb, 0x54, 0xa6, 0x2d, 0x31, 0x1a, 0xab, 0x7e, 0xf8, 0x17,
+	0xe5, 0x52, 0x2b, 0x2d, 0xcb, 0x4c, 0xa1, 0xda, 0x54, 0x0b, 0xe4, 0x63, 0xb8, 0xdb, 0xe7, 0xb2,
+	0x56, 0x24, 0xdb, 0x6d, 0x6b, 0xe0, 0xed, 0xf2, 0xd3, 0x2b, 0xc2, 0x57, 0x2a, 0xf2, 0x19, 0x84,
+	0xfb, 0xe3, 0x21, 0x93, 0xfc, 0x46, 0xaf, 0x37, 0xa1, 0xb5, 0x27, 0xc6, 0x22, 0x11, 0xaf, 0x26,
+	0x73, 0xa6, 0x3e, 0x84, 0x05, 0xbd, 0xbf, 0xf5, 0x1a, 0x69, 0x53, 0x2b, 0x92, 0x3b, 0xaa, 0xa1,
+	0x07, 0x2c, 0x19, 0x14, 0x89, 0x72, 0x43, 0xfd, 0x5e, 0xcc, 0x37, 0x97, 0x7e, 0x3f, 0x5f, 0x75,
+	0xfe, 0x38, 0x5f, 0x75, 0xfe, 0x3c, 0x5f, 0x75, 0x7e, 0xf9, 0x6b, 0xf5, 0x7f, 0x87, 0x4d, 0xfc,
+	0x3f, 0xf1, 0xe4, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x46, 0x93, 0xc0, 0xc1, 0x60, 0x0c, 0x00,
+	0x00,
 }
diff --git a/internal/private.proto b/internal/private.proto
index 57b98f62c..2e484b037 100644
--- a/internal/private.proto
+++ b/internal/private.proto
@@ -99,6 +99,7 @@ message Node {
     string ID = 1;
     URI URI = 2;
     bool IsCoordinator = 3;
+    string State = 4;
 }
 
 message NodeStateMessage {
diff --git a/server.go b/server.go
index a4cb8fc8b..5e6f61087 100644
--- a/server.go
+++ b/server.go
@@ -298,6 +298,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 		ID:            s.nodeID,
 		URI:           s.uri,
 		IsCoordinator: s.cluster.Coordinator == s.nodeID,
+		State:         nodeStateDown,
 	}
 	s.cluster.Node = node
 	if s.clusterDisabled {
@@ -561,7 +562,10 @@ func (s *Server) receiveMessage(m Message) error {
 	case *RecalculateCaches:
 		s.holder.recalculateCaches()
 	case *NodeEvent:
-		s.cluster.ReceiveEvent(obj)
+		err := s.cluster.ReceiveEvent(obj)
+		if err != nil {
+			return errors.Wrapf(err, "cluster receiving NodeEvent %v", obj)
+		}
 	case *NodeStatus:
 		s.handleRemoteStatus(obj)
 	}
diff --git a/server/server.go b/server/server.go
index 010eb2f8a..79d25d270 100644
--- a/server/server.go
+++ b/server/server.go
@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
+//
 // Package server contains the `pilosa server` subcommand which runs Pilosa
 // itself. The purpose of this package is to define an easily tested Command
 // object which handles interpreting configuration and setting up all the

From dd4685d43b4994c6bda995037fd087d00ba33b7a Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Fri, 9 Nov 2018 11:28:53 -0600
Subject: [PATCH 11/20] msg type stringer

---
 broadcast.go      | 12 ++++++++----
 msgtype_string.go | 16 ++++++++++++++++
 server.go         |  6 +++---
 3 files changed, 27 insertions(+), 7 deletions(-)
 create mode 100644 msgtype_string.go

diff --git a/broadcast.go b/broadcast.go
index 6f2245992..d18cdc937 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:generate stringer -type=msgType
+
 package pilosa
 
 import (
@@ -53,7 +55,7 @@ func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
 
 // Broadcast message types.
 const (
-	messageTypeCreateShard = iota
+	messageTypeCreateShard msgType = iota
 	messageTypeCreateIndex
 	messageTypeDeleteIndex
 	messageTypeCreateField
@@ -71,6 +73,8 @@ const (
 	messageTypeNodeStatus
 )
 
+type msgType byte
+
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
 // type info which is used by the internal messaging stuff.
 func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
@@ -79,11 +83,11 @@ func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "marshaling")
 	}
-	return append([]byte{typ}, buf...), nil
+	return append([]byte{byte(typ)}, buf...), nil
 }
 
 func getMessage(typ byte) Message {
-	switch typ {
+	switch msgType(typ) {
 	case messageTypeCreateShard:
 		return &CreateShardMessage{}
 	case messageTypeCreateIndex:
@@ -121,7 +125,7 @@ func getMessage(typ byte) Message {
 	}
 }
 
-func getMessageType(m Message) byte {
+func getMessageType(m Message) msgType {
 	switch m.(type) {
 	case *CreateShardMessage:
 		return messageTypeCreateShard
diff --git a/msgtype_string.go b/msgtype_string.go
new file mode 100644
index 000000000..d4c101bbe
--- /dev/null
+++ b/msgtype_string.go
@@ -0,0 +1,16 @@
+// Code generated by "stringer -type=msgType"; DO NOT EDIT.
+
+package pilosa
+
+import "strconv"
+
+const _msgType_name = "messageTypeCreateShardmessageTypeCreateIndexmessageTypeDeleteIndexmessageTypeCreateFieldmessageTypeDeleteFieldmessageTypeCreateViewmessageTypeDeleteViewmessageTypeClusterStatusmessageTypeResizeInstructionmessageTypeResizeInstructionCompletemessageTypeSetCoordinatormessageTypeUpdateCoordinatormessageTypeNodeStatemessageTypeRecalculateCachesmessageTypeNodeEventmessageTypeNodeStatus"
+
+var _msgType_index = [...]uint16{0, 22, 44, 66, 88, 110, 131, 152, 176, 204, 240, 265, 293, 313, 341, 361, 382}
+
+func (i msgType) String() string {
+	if i >= msgType(len(_msgType_index)-1) {
+		return "msgType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _msgType_name[_msgType_index[i]:_msgType_index[i+1]]
+}
diff --git a/server.go b/server.go
index 5e6f61087..fbbf3fa42 100644
--- a/server.go
+++ b/server.go
@@ -580,7 +580,7 @@ func (s *Server) SendSync(m Message) error {
 	if err != nil {
 		return fmt.Errorf("marshaling message: %v", err)
 	}
-	msg = append([]byte{getMessageType(m)}, msg...)
+	msg = append([]byte{byte(getMessageType(m))}, msg...)
 	for _, node := range s.cluster.nodes {
 		node := node
 		s.logger.Printf("SendSync to: %s", node.URI)
@@ -604,12 +604,12 @@ func (s *Server) SendAsync(m Message) error {
 
 // SendTo represents an implementation of Broadcaster.
 func (s *Server) SendTo(to *Node, m Message) error {
-	s.logger.Printf("SendTo: %s", to.URI)
+	s.logger.Printf("SendTo: %s, type: %s", to.URI, getMessageType(m))
 	msg, err := s.serializer.Marshal(m)
 	if err != nil {
 		return fmt.Errorf("marshaling message: %v", err)
 	}
-	msg = append([]byte{getMessageType(m)}, msg...)
+	msg = append([]byte{byte(getMessageType(m))}, msg...)
 	return s.defaultClient.SendMessage(context.Background(), &to.URI, msg)
 }
 

From 8cd53bf2c2a41f21864ed3d3580804f046d9c2fd Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Fri, 9 Nov 2018 11:34:32 -0600
Subject: [PATCH 12/20] Revert "msg type stringer"

This reverts commit dd4685d43b4994c6bda995037fd087d00ba33b7a.
---
 broadcast.go      | 12 ++++--------
 msgtype_string.go | 16 ----------------
 server.go         |  6 +++---
 3 files changed, 7 insertions(+), 27 deletions(-)
 delete mode 100644 msgtype_string.go

diff --git a/broadcast.go b/broadcast.go
index d18cdc937..6f2245992 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//go:generate stringer -type=msgType
-
 package pilosa
 
 import (
@@ -55,7 +53,7 @@ func (nopBroadcaster) SendTo(*Node, Message) error { return nil }
 
 // Broadcast message types.
 const (
-	messageTypeCreateShard msgType = iota
+	messageTypeCreateShard = iota
 	messageTypeCreateIndex
 	messageTypeDeleteIndex
 	messageTypeCreateField
@@ -73,8 +71,6 @@ const (
 	messageTypeNodeStatus
 )
 
-type msgType byte
-
 // MarshalInternalMessage serializes the pilosa message and adds pilosa internal
 // type info which is used by the internal messaging stuff.
 func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
@@ -83,11 +79,11 @@ func MarshalInternalMessage(m Message, s Serializer) ([]byte, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "marshaling")
 	}
-	return append([]byte{byte(typ)}, buf...), nil
+	return append([]byte{typ}, buf...), nil
 }
 
 func getMessage(typ byte) Message {
-	switch msgType(typ) {
+	switch typ {
 	case messageTypeCreateShard:
 		return &CreateShardMessage{}
 	case messageTypeCreateIndex:
@@ -125,7 +121,7 @@ func getMessage(typ byte) Message {
 	}
 }
 
-func getMessageType(m Message) msgType {
+func getMessageType(m Message) byte {
 	switch m.(type) {
 	case *CreateShardMessage:
 		return messageTypeCreateShard
diff --git a/msgtype_string.go b/msgtype_string.go
deleted file mode 100644
index d4c101bbe..000000000
--- a/msgtype_string.go
+++ /dev/null
@@ -1,16 +0,0 @@
-// Code generated by "stringer -type=msgType"; DO NOT EDIT.
-
-package pilosa
-
-import "strconv"
-
-const _msgType_name = "messageTypeCreateShardmessageTypeCreateIndexmessageTypeDeleteIndexmessageTypeCreateFieldmessageTypeDeleteFieldmessageTypeCreateViewmessageTypeDeleteViewmessageTypeClusterStatusmessageTypeResizeInstructionmessageTypeResizeInstructionCompletemessageTypeSetCoordinatormessageTypeUpdateCoordinatormessageTypeNodeStatemessageTypeRecalculateCachesmessageTypeNodeEventmessageTypeNodeStatus"
-
-var _msgType_index = [...]uint16{0, 22, 44, 66, 88, 110, 131, 152, 176, 204, 240, 265, 293, 313, 341, 361, 382}
-
-func (i msgType) String() string {
-	if i >= msgType(len(_msgType_index)-1) {
-		return "msgType(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-	return _msgType_name[_msgType_index[i]:_msgType_index[i+1]]
-}
diff --git a/server.go b/server.go
index fbbf3fa42..5e6f61087 100644
--- a/server.go
+++ b/server.go
@@ -580,7 +580,7 @@ func (s *Server) SendSync(m Message) error {
 	if err != nil {
 		return fmt.Errorf("marshaling message: %v", err)
 	}
-	msg = append([]byte{byte(getMessageType(m))}, msg...)
+	msg = append([]byte{getMessageType(m)}, msg...)
 	for _, node := range s.cluster.nodes {
 		node := node
 		s.logger.Printf("SendSync to: %s", node.URI)
@@ -604,12 +604,12 @@ func (s *Server) SendAsync(m Message) error {
 
 // SendTo represents an implementation of Broadcaster.
 func (s *Server) SendTo(to *Node, m Message) error {
-	s.logger.Printf("SendTo: %s, type: %s", to.URI, getMessageType(m))
+	s.logger.Printf("SendTo: %s", to.URI)
 	msg, err := s.serializer.Marshal(m)
 	if err != nil {
 		return fmt.Errorf("marshaling message: %v", err)
 	}
-	msg = append([]byte{byte(getMessageType(m))}, msg...)
+	msg = append([]byte{getMessageType(m)}, msg...)
 	return s.defaultClient.SendMessage(context.Background(), &to.URI, msg)
 }
 

From 90c5f64b194f827bb68fb23301f6c793c1d1c714 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Mon, 12 Nov 2018 14:01:57 -0600
Subject: [PATCH 13/20] filter memberlist debug and info logs, use t.Log
 instead of fmt in cluster tests

---
 gossip/gossip.go                      | 15 ++++++++++++++-
 internal/clustertests/cluster_test.go | 10 ++++------
 server/server.go                      | 24 +++++++++++++++++++++++-
 3 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/gossip/gossip.go b/gossip/gossip.go
index 789ed50b0..ecd663ecf 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net"
@@ -49,6 +50,7 @@ type memberSet struct {
 	Logger pilosa.Logger
 
 	logger    *log.Logger
+	logOutput io.Writer
 	transport *Transport
 
 	eventReceiver *eventReceiver
@@ -156,6 +158,13 @@ func WithLogger(logger *log.Logger) memberSetOption {
 	}
 }
 
+func WithLogOutput(o io.Writer) memberSetOption {
+	return func(g *memberSet) error {
+		g.logOutput = o
+		return nil
+	}
+}
+
 // NewMemberSet returns a new instance of GossipMemberSet based on options.
 func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*memberSet, error) {
 	host := api.Node().URI.Host
@@ -220,7 +229,11 @@ func NewMemberSet(cfg Config, api *pilosa.API, options ...memberSetOption) (*mem
 	conf.Delegate = g
 	conf.SecretKey = gossipKey
 	conf.Events = ger
-	conf.Logger = g.logger
+	if g.logOutput != nil {
+		conf.LogOutput = g.logOutput
+	} else {
+		conf.Logger = g.logger
+	}
 
 	g.config = &config{
 		memberlistConfig: conf,
diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go
index 21c9e35de..7ced51042 100644
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@@ -1,7 +1,6 @@
 package clustertest
 
 import (
-	"fmt"
 	"io"
 	"os"
 	"os/exec"
@@ -19,6 +18,7 @@ func TestClusterStuff(t *testing.T) {
 	cli := getPilosaClient(t)
 
 	t.Run("long pause", func(t *testing.T) {
+
 		idx := pilosa.NewIndex("testidx")
 		err := cli.CreateIndex(idx)
 		if err != nil {
@@ -52,7 +52,7 @@ func TestClusterStuff(t *testing.T) {
 		pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s")
 		pcmd.Stdout = os.Stdout
 		pcmd.Stderr = os.Stderr
-		fmt.Println("pausing pilosa3 for 10s")
+		t.Log("pausing pilosa3 for 10s")
 		err = pcmd.Start()
 		if err != nil {
 			t.Fatalf("starting pumba command: %v", err)
@@ -62,9 +62,9 @@ func TestClusterStuff(t *testing.T) {
 			t.Fatalf("waiting on pumba pause cmd: %v", err)
 		}
 		// TODO change the sleep to wait for status to return to NORMAL or timeout once we have Status.State support in go-pilosa
-		fmt.Println("done with pause, waiting for stability")
+		t.Log("done with pause, waiting for stability")
 		time.Sleep(time.Second * 3)
-		fmt.Println("done waiting")
+		t.Log("done waiting for stability")
 
 		r, err = cli.Query(idx.Count(f.Row(0)))
 		if err != nil {
@@ -73,8 +73,6 @@ func TestClusterStuff(t *testing.T) {
 			t.Fatalf("count after import is %d", r.Result().Count())
 		}
 
-		fmt.Println("at the bottom")
-
 	})
 
 	down := exec.Command("/pumba", "stop", "clustertests_pilosa3_1", "clustertests_pilosa2_1", "clustertests_pilosa1_1")
diff --git a/server/server.go b/server/server.go
index 79d25d270..6cdc43883 100644
--- a/server/server.go
+++ b/server/server.go
@@ -20,6 +20,7 @@
 package server
 
 import (
+	"bytes"
 	"crypto/tls"
 	"io"
 	"log"
@@ -336,7 +337,7 @@ func (m *Command) setupNetworking() error {
 	gossipMemberSet, err := gossip.NewMemberSet(
 		m.Config.Gossip,
 		m.API,
-		gossip.WithLogger(m.logger.Logger()),
+		gossip.WithLogOutput(&filteredWriter{logOutput: m.logOutput, v: m.Config.Verbose}),
 		gossip.WithTransport(m.gossipTransport),
 	)
 	if err != nil {
@@ -407,3 +408,24 @@ func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err erro
 
 	return ln, nil
 }
+
+type filteredWriter struct {
+	v         bool
+	logOutput io.Writer
+}
+
+// Write forwards the write to logOutput if verbose is true, or it doesn't
+// contain [DEBUG] or [INFO]. This implementation isn't technically correct
+// since Write could be called with only part of a log line, but I don't think
+// that actually happens, so until it becomes a problem, I don't think it's
+// worth dealing with the extra complexity. (jaffee)
+func (f *filteredWriter) Write(p []byte) (n int, err error) {
+	if bytes.Contains(p, []byte("[DEBUG]")) || bytes.Contains(p, []byte("[INFO]")) {
+		if f.v {
+			return f.logOutput.Write(p)
+		}
+	} else {
+		return f.logOutput.Write(p)
+	}
+	return len(p), nil
+}

From 37ac8b7a93adea7325c6b728bd7e315f9d0253f6 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Mon, 12 Nov 2018 17:44:23 -0600
Subject: [PATCH 14/20] better use of docker-compose opts per code review

---
 Makefile                              | 11 +++++------
 internal/clustertests/cluster_test.go |  9 ---------
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/Makefile b/Makefile
index 9cf4cffaf..34f97965e 100644
--- a/Makefile
+++ b/Makefile
@@ -73,16 +73,15 @@ release: check-clean
 # make changes to Pilosa, you'll want to run clustertests-build to rebuild the
 # pilosa image.
 clustertests:
-	cd internal/clustertests;\
-	docker-compose down;\
-	docker-compose up;
+	docker-compose -f internal/clustertests/docker-compose.yml down
+	docker-compose -f internal/clustertests/docker-compose.yml build client1
+	docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1
 
 
 # Like clustertests, but rebuilds all images.
 clustertests-build:
-	cd internal/clustertests;\
-	docker-compose down;\
-	docker-compose up --build;
+	docker-compose -f internal/clustertests/docker-compose.yml down
+	docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 --build
 
 # Create prerelease builds
 prerelease: vendor
diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go
index 7ced51042..402a8c14c 100644
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@@ -18,7 +18,6 @@ func TestClusterStuff(t *testing.T) {
 	cli := getPilosaClient(t)
 
 	t.Run("long pause", func(t *testing.T) {
-
 		idx := pilosa.NewIndex("testidx")
 		err := cli.CreateIndex(idx)
 		if err != nil {
@@ -72,16 +71,8 @@ func TestClusterStuff(t *testing.T) {
 		} else if r.Result().Count() != 1000 {
 			t.Fatalf("count after import is %d", r.Result().Count())
 		}
-
 	})
 
-	down := exec.Command("/pumba", "stop", "clustertests_pilosa3_1", "clustertests_pilosa2_1", "clustertests_pilosa1_1")
-	down.Stdout = os.Stdout
-	down.Stderr = os.Stderr
-	err := down.Run()
-	if err != nil {
-		t.Logf("stopping Pilosa: %v", err)
-	}
 }
 
 // Utils

From 0d4a46af97f53bc719daebc902c8e3ce0cf131e6 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 13 Nov 2018 09:24:35 -0600
Subject: [PATCH 15/20] use internal client instead of go-pilosa, use ADD
 instead of wget

---
 Dockerfile-withgo                     |   2 +-
 internal/clustertests/cluster_test.go | 120 +++++++-------------------
 2 files changed, 30 insertions(+), 92 deletions(-)

diff --git a/Dockerfile-withgo b/Dockerfile-withgo
index 834634b13..c60fda576 100644
--- a/Dockerfile-withgo
+++ b/Dockerfile-withgo
@@ -11,7 +11,7 @@ RUN cd /go/src/github.com/pilosa/pilosa \
     && CGO_ENABLED=0 make install-dep install FLAGS="-a"
 
 # download pumba for fault injection
-RUN wget https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 -O /pumba
+ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
 RUN chmod +x /pumba
 
 RUN cp /go/bin/pilosa /pilosa
diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go
index 402a8c14c..f01835c10 100644
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@@ -1,51 +1,54 @@
 package clustertest
 
 import (
-	"io"
+	"context"
 	"os"
 	"os/exec"
 	"testing"
 	"time"
 
-	"github.com/pilosa/go-pilosa"
-	pi "github.com/pilosa/pilosa"
+	"github.com/pilosa/pilosa"
+	picli "github.com/pilosa/pilosa/http"
 )
 
 func TestClusterStuff(t *testing.T) {
 	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
 		t.Skip()
 	}
-	cli := getPilosaClient(t)
+	cli, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil))
+	if err != nil {
+		t.Fatalf("getting client: %v", err)
+	}
 
 	t.Run("long pause", func(t *testing.T) {
-		idx := pilosa.NewIndex("testidx")
-		err := cli.CreateIndex(idx)
+		err := cli.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{})
 		if err != nil {
 			t.Fatalf("creating index: %v", err)
 		}
-		f := idx.Field("testf", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10))
-		err = cli.CreateField(f)
+		err = cli.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100})
 		if err != nil {
 			t.Fatalf("creating field: %v", err)
 		}
 
-		data := make([]pilosa.Column, 1000)
-		for i := range data {
-			data[i].RowID = 0
-			data[i].ColumnID = uint64((i/10)*pi.ShardWidth + i%10)
+		data := make([]pilosa.Bit, 10)
+		for i := 0; i < 1000; i++ {
+			data[i%10].RowID = 0
+			data[i%10].ColumnID = uint64((i/10)*pilosa.ShardWidth + i%10)
+			shard := uint64(i / 10)
+			if i%10 == 9 {
+				err = cli.Import(context.Background(), "testidx", "testf", shard, data)
+				if err != nil {
+					t.Fatalf("importing: %v", err)
+				}
+			}
 		}
 
-		err = cli.ImportField(f, &colIterator{cols: data}, pilosa.OptImportBatchSize(1000))
-		if err != nil {
-			t.Fatalf("importing: %v", err)
-		}
-
-		r, err := cli.Query(idx.Count(f.Row(0)))
+		r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
 		if err != nil {
 			t.Fatalf("count querying: %v", err)
 		}
-		if r.Result().Count() != 1000 {
-			t.Fatalf("count after import is %d", r.Result().Count())
+		if r.Results[0].(uint64) != 1000 {
+			t.Fatalf("count after import is %d", r.Results[0].(uint64))
 		}
 
 		pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s")
@@ -60,84 +63,19 @@ func TestClusterStuff(t *testing.T) {
 		if err != nil {
 			t.Fatalf("waiting on pumba pause cmd: %v", err)
 		}
-		// TODO change the sleep to wait for status to return to NORMAL or timeout once we have Status.State support in go-pilosa
+
+		// TODO change the sleep to wait for status to return to NORMAL - need support in internal client for getting status
 		t.Log("done with pause, waiting for stability")
 		time.Sleep(time.Second * 3)
 		t.Log("done waiting for stability")
 
-		r, err = cli.Query(idx.Count(f.Row(0)))
+		r, err = cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
 		if err != nil {
 			t.Fatalf("count querying: %v", err)
-		} else if r.Result().Count() != 1000 {
-			t.Fatalf("count after import is %d", r.Result().Count())
+		}
+		if r.Results[0].(uint64) != 1000 {
+			t.Fatalf("count after import is %d", r.Results[0].(uint64))
 		}
 	})
 
 }
-
-// Utils
-
-func getPilosaClient(t *testing.T) *pilosa.Client {
-	cli, err := pilosa.NewClient("pilosa1:10101")
-	if err != nil {
-		time.Sleep(time.Millisecond * 40)
-	}
-	time.Sleep(time.Second * 2)
-	// TODO uncomment the following once we get the version of go-pilosa that has the State field on Status.
-	// start := time.Now()
-	// for i := 0; true; i++ {
-	// 	s, err := cli.Status()
-	// 	if i > 800 {
-	// 		t.Fatalf("couldn't connect to cluster after %d attempts and %v: state: %s, err: %v", i, time.Since(start), s.State, err)
-	// 	}
-	// 	if err != nil {
-	// 		time.Sleep(time.Millisecond * 40)
-	// 		continue
-	// 	}
-	// 	if s.State == "NORMAL" {
-	// 		break
-	// 	} else {
-	// 		time.Sleep(time.Millisecond * 40)
-	// 	}
-	// }
-
-	return cli
-}
-
-type colIterator struct {
-	cols []pilosa.Column
-	i    uint
-}
-
-func (c *colIterator) NextRecord() (pilosa.Record, error) {
-	if int(c.i) >= len(c.cols) {
-		return nil, io.EOF
-	}
-	c.i++
-	return c.cols[c.i-1], nil
-}
-
-func TestColIterator(t *testing.T) {
-	data := make([]pilosa.Column, 3)
-	for i := range data {
-		data[i].RowID = 0
-		data[i].ColumnID = uint64((i/10)*pi.ShardWidth + i%10)
-	}
-
-	ci := colIterator{cols: data}
-	col := pilosa.Column{}
-	if rec, err := ci.NextRecord(); rec != col {
-		t.Fatalf("first record wrong: %v, err: %v", rec, err)
-	}
-	col.ColumnID = 1
-	if rec, err := ci.NextRecord(); rec != col || err != nil {
-		t.Fatalf("second record wrong: %v, err: %v", rec, err)
-	}
-	col.ColumnID = 2
-	if rec, err := ci.NextRecord(); rec != col || err != nil {
-		t.Fatalf("third record wrong: %v, err: %v", rec, err)
-	}
-	if rec, err := ci.NextRecord(); err != io.EOF {
-		t.Fatalf("should be EOF, but got %v, err: %v", rec, err)
-	}
-}

From bc8b9912203463cfba46b67f9b5c0ee9d3a154c5 Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 13 Nov 2018 09:31:16 -0600
Subject: [PATCH 16/20] rename Dockerfile-withgo to Dockerfile-clustertests

---
 Dockerfile-withgo => Dockerfile-clustertests | 0
 internal/clustertests/docker-compose.yml     | 7 ++++---
 2 files changed, 4 insertions(+), 3 deletions(-)
 rename Dockerfile-withgo => Dockerfile-clustertests (100%)

diff --git a/Dockerfile-withgo b/Dockerfile-clustertests
similarity index 100%
rename from Dockerfile-withgo
rename to Dockerfile-clustertests
diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml
index 7196ea0eb..8586eb08f 100644
--- a/internal/clustertests/docker-compose.yml
+++ b/internal/clustertests/docker-compose.yml
@@ -3,7 +3,7 @@ services:
   pilosa1:
     build:
       context: ../..
-      dockerfile: Dockerfile-withgo
+      dockerfile: Dockerfile-clustertests
     image: ptest
     ports:
       - "33455:10101"
@@ -17,7 +17,7 @@ services:
   pilosa2:
     build:
       context: ../..
-      dockerfile: Dockerfile-withgo
+      dockerfile: Dockerfile-clustertests
     image: ptest
     ports:
       - "33456:10101"
@@ -29,7 +29,8 @@ services:
       - "/pilosa server --bind pilosa2:10101"
   pilosa3:
     build:
-      context: .
+      context: ../..
+      dockerfile: Dockerfile-clustertests
     image: ptest
     ports:
       - "33457:10101"

From 26cd50339309dc9981d2ead1116c0c73c368025f Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 13 Nov 2018 09:51:22 -0600
Subject: [PATCH 17/20] try to run clustertests in CI

---
 .circleci/config.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index bd1ca86cd..131ee934e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -22,6 +22,7 @@ jobs:
       - persist_to_workspace:
           root: .
           paths: "*"
+      - setup_remote_docker
   linter:
     <<: *defaults
     steps:
@@ -44,6 +45,11 @@ jobs:
     <<: *base-test
     environment:
       GOARCH: 386
+  cluster-tests:
+    <<: *defaults
+    sets:
+      - *fast-checkout
+      - run: make clustertests-build
   prerelease:
     <<: *base-test
     steps:

From 08431f6e762bac0b691be2e2f06b1200dd29784f Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 13 Nov 2018 09:53:44 -0600
Subject: [PATCH 18/20] iterate on ci config

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 131ee934e..3e302ecac 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -47,7 +47,7 @@ jobs:
       GOARCH: 386
   cluster-tests:
     <<: *defaults
-    sets:
+    steps:
       - *fast-checkout
       - run: make clustertests-build
   prerelease:

From a9d108200b75bb92d73f322332b5de82e6a1493f Mon Sep 17 00:00:00 2001
From: Matt Jaffee <matthew.jaffee@gmail.com>
Date: Tue, 13 Nov 2018 11:52:08 -0600
Subject: [PATCH 19/20] update circle ci config with cody's feedback

---
 .circleci/config.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3e302ecac..605990e5b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -22,7 +22,6 @@ jobs:
       - persist_to_workspace:
           root: .
           paths: "*"
-      - setup_remote_docker
   linter:
     <<: *defaults
     steps:
@@ -49,6 +48,7 @@ jobs:
     <<: *defaults
     steps:
       - *fast-checkout
+      - setup_remote_docker
       - run: make clustertests-build
   prerelease:
     <<: *base-test
@@ -105,6 +105,9 @@ workflows:
       - test-golang-1.10-386:
           requires:
             - build
+      - cluster-tests:
+          requires:
+            - build
       - prerelease:
           requires:
             - linter

From 1cd7ebdd2c9889998cdf7fd4e1157b7db9028fa6 Mon Sep 17 00:00:00 2001
From: Cody Soyland <cody@soyland.org>
Date: Tue, 13 Nov 2018 12:32:43 -0600
Subject: [PATCH 20/20] Remove TravisCI, add CircleCI shield

---
 .travis.yml     | 52 -------------------------------------------------
 CONTRIBUTING.md |  2 +-
 README.md       |  2 +-
 3 files changed, 2 insertions(+), 54 deletions(-)
 delete mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 6c77fe0fc..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-language: go
-go:
-  - "1.10" # Use string, as 1.10==1.1 if interpreted as float.
-  - master
-env:
-  global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
-    - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA="
-    - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
-  matrix:
-    - GOARCH=386
-    - GOARCH=386 ENTERPRISE=1
-    - GOARCH=amd64
-    - GOARCH=amd64 ENTERPRISE=1
-cache:
-  directories:
-    vendor
-install:
-  - make -B install-dep vendor
-script: make test
-jobs:
-  include:
-    - stage: metalinter
-      install:
-        - make -B install-dep vendor install-gometalinter
-      script: make gometalinter
-      env:
-        - GOARCH=amd64
-    - stage: deploy
-      script: skip
-      go: "1.10"
-      env:
-        - GOARCH=amd64
-      deploy:
-        - provider: script
-          script: pip install awscli --user `whoami` && make -B prerelease
-          skip_cleanup: true
-          on:
-            all_branches: true
-            repo: pilosa/pilosa
-            tags: false
-stages:
-  - metalinter
-  - test
-  - deploy
-matrix:
-  # Excluding or allowing failures on non-primary matrix configurations due to long running times.
-  fast_finish: true
-  allow_failures:
-    - go: master
-notifications:
-  slack:
-    secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 2ad9ab13d..3939e927e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -22,7 +22,7 @@ If you want to help but you aren't sure where to start, check out our [github la
 
 ### Development Environment
 
-- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [travis file](../master/.travis.yml) for the most up-to-date information.
+- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information.
 
 - Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH).
 
diff --git a/README.md b/README.md
index 7c1c7cfda..8ccae2b91 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
     </a>
 </p>
 
-[![Build Status](https://travis-ci.org/pilosa/pilosa.svg?branch=master)](https://travis-ci.org/pilosa/pilosa)
+[![CircleCI](https://circleci.com/gh/pilosa/pilosa/tree/master.svg?style=shield)](https://circleci.com/gh/pilosa/pilosa/tree/master)
 [![GoDoc](https://godoc.org/github.com/pilosa/pilosa?status.svg)](https://godoc.org/github.com/pilosa/pilosa)
 [![Go Report Card](https://goreportcard.com/badge/github.com/pilosa/pilosa)](https://goreportcard.com/report/github.com/pilosa/pilosa)
 [![license](https://img.shields.io/github/license/pilosa/pilosa.svg)](https://github.com/pilosa/pilosa/blob/master/LICENSE)