From 74bf4731f5ca4060076a83b44fdb22b57f3ce7f1 Mon Sep 17 00:00:00 2001
From: Travis <travis@pilosa.com>
Date: Fri, 21 Apr 2017 18:27:46 -0500
Subject: [PATCH 1/5] WIP: Implement NodeState as an attribute of *Node
 NodeState is shared among nodes in the cluster (via gossip in a gossip
 implementation) and cached locally in Cluster.Nodes in order to be available
 to the /status endpoint.

---
 cluster.go             |  47 ++++++--
 cluster_test.go        |  12 +-
 gossip/gossip.go       |  22 ++--
 handler.go             |  15 ++-
 internal/private.pb.go | 248 ++++++++++++++++++++++++++++++++---------
 internal/private.proto |   6 +-
 server.go              |  51 +++++++--
 7 files changed, 305 insertions(+), 96 deletions(-)

diff --git a/cluster.go b/cluster.go
index b9d799aec..f9f22464f 100644
--- a/cluster.go
+++ b/cluster.go
@@ -3,6 +3,8 @@ package pilosa
 import (
 	"encoding/binary"
 	"hash/fnv"
+
+	"github.com/pilosa/pilosa/internal"
 )
 
 const (
@@ -12,15 +14,30 @@ const (
 	// DefaultReplicaN is the default number of replicas per partition.
 	DefaultReplicaN = 1
 
-	// HealthStatus is the return value of the /health endpoint for a node in the cluster.
-	HealthStatusUp   = "UP"
-	HealthStatusDown = "DOWN"
+	// NodeState represents node state returned in /status endpoint for a node in the cluster.
+	NodeStateUp   = "UP"
+	NodeStateDown = "DOWN"
 )
 
 // Node represents a node in the cluster.
 type Node struct {
 	Host         string `json:"host"`
 	InternalHost string `json:"internalHost"`
+
+	status *internal.NodeStatus `json:"state"`
+}
+
+// SetStatus sets the NodeStatus.
+func (n *Node) SetStatus(s *internal.NodeStatus) {
+	n.status = s
+}
+
+// SetState sets the Node.status.state.
+func (n *Node) SetState(s string) {
+	if n.status == nil {
+		n.status = &internal.NodeStatus{}
+	}
+	n.status.State = s
 }
 
 // Nodes represents a list of nodes.
@@ -120,21 +137,37 @@ func (c *Cluster) NodeSetHosts() []string {
 	return a
 }
 
-// Health returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value.
-func (c *Cluster) Health() map[string]string {
+// NodeStates returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value.
+func (c *Cluster) NodeStates() map[string]string {
 	h := make(map[string]string)
 	for _, n := range c.Nodes {
-		h[n.Host] = HealthStatusDown
+		h[n.Host] = NodeStateDown
 	}
 	// we are assuming that NodeSetHosts is a subset of c.Nodes
 	for _, m := range c.NodeSetHosts() {
 		if _, ok := h[m]; ok {
-			h[m] = HealthStatusUp
+			h[m] = NodeStateUp
 		}
 	}
 	return h
 }
 
+// State returns the internal ClusterState representation.
+func (c *Cluster) Status() *internal.ClusterStatus {
+	return &internal.ClusterStatus{
+		Nodes: encodeClusterStatus(c.Nodes),
+	}
+}
+
+// encodeClusterStatus converts a into its internal representation.
+func encodeClusterStatus(a []*Node) []*internal.NodeStatus {
+	other := make([]*internal.NodeStatus, len(a))
+	for i := range a {
+		other[i] = a[i].status
+	}
+	return other
+}
+
 // NodeByHost returns a node reference by host.
 func (c *Cluster) NodeByHost(host string) *Node {
 	for _, n := range c.Nodes {
diff --git a/cluster_test.go b/cluster_test.go
index e561223fb..4c96cd5d9 100644
--- a/cluster_test.go
+++ b/cluster_test.go
@@ -89,7 +89,7 @@ func TestCluster_NodeSetHosts(t *testing.T) {
 }
 
 // Ensure cluster can compare its Nodes and Members
-func TestCluster_Health(t *testing.T) {
+func TestCluster_NodeStates(t *testing.T) {
 	c := pilosa.Cluster{
 		Nodes: []*pilosa.Node{
 			{Host: "serverA:1000"},
@@ -109,12 +109,12 @@ func TestCluster_Health(t *testing.T) {
 	}
 
 	// Verify a DOWN node is reported, and extraneous nodes are ignored
-	if a := c.Health(); !reflect.DeepEqual(a, map[string]string{
-		"serverA:1000": pilosa.HealthStatusUp,
-		"serverB:1000": pilosa.HealthStatusDown,
-		"serverC:1000": pilosa.HealthStatusUp,
+	if a := c.NodeStates(); !reflect.DeepEqual(a, map[string]string{
+		"serverA:1000": pilosa.NodeStateUp,
+		"serverB:1000": pilosa.NodeStateDown,
+		"serverC:1000": pilosa.NodeStateUp,
 	}) {
-		t.Fatalf("unexpected health: %s", spew.Sdump(a))
+		t.Fatalf("unexpected node state: %s", spew.Sdump(a))
 	}
 }
 
diff --git a/gossip/gossip.go b/gossip/gossip.go
index 772892926..c3ded776a 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -14,14 +14,6 @@ import (
 	"github.com/pilosa/pilosa/internal"
 )
 
-// StateHandler specifies two methods which an object must implement to share
-// state in the cluster. These are used by the GossipNodeSet to implement the
-// LocalState and MergeRemoteState methods of memberlist.Delegate
-type StateHandler interface {
-	LocalState() (proto.Message, error)
-	HandleRemoteState(proto.Message) error
-}
-
 // GossipNodeSet represents a gossip implementation of NodeSet using memberlist
 // GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster
 // GossipNodeSet also represents an implementation of memberlist.Delegate
@@ -31,8 +23,8 @@ type GossipNodeSet struct {
 
 	broadcasts *memberlist.TransmitLimitedQueue
 
-	stateHandler StateHandler
-	config       *GossipConfig
+	statusHandler pilosa.StatusHandler
+	config        *GossipConfig
 
 	// The writer for any logging.
 	LogOutput io.Writer
@@ -89,7 +81,7 @@ type GossipConfig struct {
 }
 
 // NewGossipNodeSet returns a new instance of GossipNodeSet.
-func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh StateHandler) *GossipNodeSet {
+func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh pilosa.StatusHandler) *GossipNodeSet {
 	g := &GossipNodeSet{
 		LogOutput: os.Stderr,
 	}
@@ -106,7 +98,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed
 	g.config.memberlistConfig.AdvertisePort = gossipPort
 	g.config.memberlistConfig.Delegate = g
 
-	g.stateHandler = sh
+	g.statusHandler = sh
 
 	return g
 }
@@ -176,7 +168,7 @@ func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte {
 }
 
 func (g *GossipNodeSet) LocalState(join bool) []byte {
-	pb, err := g.stateHandler.LocalState()
+	pb, err := g.statusHandler.LocalStatus()
 	if err != nil {
 		g.logger().Printf("error getting local state, err=%s", err)
 		return []byte{}
@@ -193,12 +185,12 @@ func (g *GossipNodeSet) LocalState(join bool) []byte {
 
 func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) {
 	// Unmarshal nodestate data.
-	var pb internal.NodeState
+	var pb internal.NodeStatus
 	if err := proto.Unmarshal(buf, &pb); err != nil {
 		g.logger().Printf("error unmarshalling nodestate data, err=%s", err)
 		return
 	}
-	err := g.stateHandler.HandleRemoteState(&pb)
+	err := g.statusHandler.HandleRemoteStatus(&pb)
 	if err != nil {
 		g.logger().Printf("merge state error: %s", err)
 	}
diff --git a/handler.go b/handler.go
index 904db8ce5..77dcb5700 100644
--- a/handler.go
+++ b/handler.go
@@ -27,8 +27,9 @@ import (
 
 // Handler represents an HTTP handler.
 type Handler struct {
-	Holder      *Holder
-	Broadcaster Broadcaster
+	Holder        *Holder
+	Broadcaster   Broadcaster
+	StatusHandler StatusHandler
 
 	// Local hostname & cluster configuration.
 	Host    string
@@ -85,6 +86,7 @@ func NewRouter(handler *Handler) *mux.Router {
 	router.HandleFunc("/nodes", handler.handleGetNodes).Methods("GET")
 	router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
 	router.HandleFunc("/slices/max", handler.handleGetSliceMax).Methods("GET")
+	router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
 	router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
 
 	// TODO: Apply MethodNotAllowed statuses to all endpoints.
@@ -116,8 +118,13 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
 
 // handleGetStatus handles GET /status requests.
 func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
+	status, err := h.StatusHandler.ClusterStatus()
+	if err != nil {
+		h.logger().Printf("cluster status error: %s", err)
+		return
+	}
 	if err := json.NewEncoder(w).Encode(getStatusResponse{
-		Health: h.Cluster.Health(),
+		Status: status,
 	}); err != nil {
 		h.logger().Printf("write status response error: %s", err)
 	}
@@ -128,7 +135,7 @@ type getSchemaResponse struct {
 }
 
 type getStatusResponse struct {
-	Health map[string]string `json:"health"`
+	Status proto.Message `json:"status"`
 }
 
 // handlePostQuery handles /query requests.
diff --git a/internal/private.pb.go b/internal/private.pb.go
index 0e86dd7b6..a41df4788 100644
--- a/internal/private.pb.go
+++ b/internal/private.pb.go
@@ -23,7 +23,8 @@
 		DeleteFrameMessage
 		Frame
 		Index
-		NodeState
+		NodeStatus
+		ClusterStatus
 */
 package internal
 
@@ -231,24 +232,40 @@ func (m *Index) GetFrames() []*Frame {
 	return nil
 }
 
-type NodeState struct {
+type NodeStatus struct {
 	Host    string   `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"`
 	State   string   `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
 	Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"`
 }
 
-func (m *NodeState) Reset()                    { *m = NodeState{} }
-func (m *NodeState) String() string            { return proto.CompactTextString(m) }
-func (*NodeState) ProtoMessage()               {}
-func (*NodeState) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} }
+func (m *NodeStatus) Reset()                    { *m = NodeStatus{} }
+func (m *NodeStatus) String() string            { return proto.CompactTextString(m) }
+func (*NodeStatus) ProtoMessage()               {}
+func (*NodeStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} }
 
-func (m *NodeState) GetIndexes() []*Index {
+func (m *NodeStatus) GetIndexes() []*Index {
 	if m != nil {
 		return m.Indexes
 	}
 	return nil
 }
 
+type ClusterStatus struct {
+	Nodes []*NodeStatus `protobuf:"bytes,1,rep,name=Nodes" json:"Nodes,omitempty"`
+}
+
+func (m *ClusterStatus) Reset()                    { *m = ClusterStatus{} }
+func (m *ClusterStatus) String() string            { return proto.CompactTextString(m) }
+func (*ClusterStatus) ProtoMessage()               {}
+func (*ClusterStatus) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{15} }
+
+func (m *ClusterStatus) GetNodes() []*NodeStatus {
+	if m != nil {
+		return m.Nodes
+	}
+	return nil
+}
+
 func init() {
 	proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta")
 	proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta")
@@ -264,7 +281,8 @@ func init() {
 	proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage")
 	proto.RegisterType((*Frame)(nil), "internal.Frame")
 	proto.RegisterType((*Index)(nil), "internal.Index")
-	proto.RegisterType((*NodeState)(nil), "internal.NodeState")
+	proto.RegisterType((*NodeStatus)(nil), "internal.NodeStatus")
+	proto.RegisterType((*ClusterStatus)(nil), "internal.ClusterStatus")
 }
 func (m *IndexMeta) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
@@ -780,7 +798,7 @@ func (m *Index) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }
 
-func (m *NodeState) Marshal() (dAtA []byte, err error) {
+func (m *NodeStatus) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
 	n, err := m.MarshalTo(dAtA)
@@ -790,7 +808,7 @@ func (m *NodeState) Marshal() (dAtA []byte, err error) {
 	return dAtA[:n], nil
 }
 
-func (m *NodeState) MarshalTo(dAtA []byte) (int, error) {
+func (m *NodeStatus) MarshalTo(dAtA []byte) (int, error) {
 	var i int
 	_ = i
 	var l int
@@ -822,6 +840,36 @@ func (m *NodeState) MarshalTo(dAtA []byte) (int, error) {
 	return i, nil
 }
 
+func (m *ClusterStatus) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ClusterStatus) MarshalTo(dAtA []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Nodes) > 0 {
+		for _, msg := range m.Nodes {
+			dAtA[i] = 0xa
+			i++
+			i = encodeVarintPrivate(dAtA, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(dAtA[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
+	return i, nil
+}
+
 func encodeFixed64Private(dAtA []byte, offset int, v uint64) int {
 	dAtA[offset] = uint8(v)
 	dAtA[offset+1] = uint8(v >> 8)
@@ -1074,7 +1122,7 @@ func (m *Index) Size() (n int) {
 	return n
 }
 
-func (m *NodeState) Size() (n int) {
+func (m *NodeStatus) Size() (n int) {
 	var l int
 	_ = l
 	l = len(m.Host)
@@ -1094,6 +1142,18 @@ func (m *NodeState) Size() (n int) {
 	return n
 }
 
+func (m *ClusterStatus) Size() (n int) {
+	var l int
+	_ = l
+	if len(m.Nodes) > 0 {
+		for _, e := range m.Nodes {
+			l = e.Size()
+			n += 1 + l + sovPrivate(uint64(l))
+		}
+	}
+	return n
+}
+
 func sovPrivate(x uint64) (n int) {
 	for {
 		n++
@@ -2899,7 +2959,7 @@ func (m *Index) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
-func (m *NodeState) Unmarshal(dAtA []byte) error {
+func (m *NodeStatus) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0
 	for iNdEx < l {
@@ -2922,10 +2982,10 @@ func (m *NodeState) Unmarshal(dAtA []byte) error {
 		fieldNum := int32(wire >> 3)
 		wireType := int(wire & 0x7)
 		if wireType == 4 {
-			return fmt.Errorf("proto: NodeState: wiretype end group for non-group")
+			return fmt.Errorf("proto: NodeStatus: wiretype end group for non-group")
 		}
 		if fieldNum <= 0 {
-			return fmt.Errorf("proto: NodeState: illegal tag %d (wire type %d)", fieldNum, wire)
+			return fmt.Errorf("proto: NodeStatus: illegal tag %d (wire type %d)", fieldNum, wire)
 		}
 		switch fieldNum {
 		case 1:
@@ -3038,6 +3098,87 @@ func (m *NodeState) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *ClusterStatus) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPrivate
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ClusterStatus: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ClusterStatus: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPrivate
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Nodes = append(m.Nodes, &NodeStatus{})
+			if err := m.Nodes[len(m.Nodes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPrivate(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPrivate
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func skipPrivate(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
@@ -3146,43 +3287,44 @@ var (
 func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
 
 var fileDescriptorPrivate = []byte{
-	// 594 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x4c,
-	0x10, 0xfe, 0x9d, 0xb8, 0xfd, 0xe3, 0x89, 0x1a, 0xd2, 0x05, 0x21, 0x53, 0xa1, 0x28, 0xda, 0x03,
-	0x0d, 0x3d, 0xe4, 0x50, 0x2e, 0x08, 0x71, 0xa8, 0x9a, 0x04, 0x35, 0x12, 0x29, 0x62, 0x53, 0x71,
-	0x66, 0x93, 0x8c, 0xc0, 0x8a, 0x63, 0x07, 0x7b, 0x93, 0x34, 0x1c, 0xb8, 0xf3, 0x06, 0x48, 0x3c,
-	0x03, 0xef, 0xc1, 0x91, 0x47, 0x40, 0xe1, 0x45, 0xd0, 0x8e, 0xd7, 0x76, 0x70, 0x29, 0x15, 0xdc,
-	0x76, 0xbe, 0x99, 0x9d, 0xef, 0x9b, 0xcf, 0xb3, 0x86, 0xbd, 0x79, 0xe4, 0x2d, 0xa5, 0xc2, 0xf6,
-	0x3c, 0x0a, 0x55, 0xc8, 0x2a, 0x5e, 0xa0, 0x30, 0x0a, 0xa4, 0xcf, 0x5f, 0x80, 0xd3, 0x0f, 0x26,
-	0x78, 0x39, 0x40, 0x25, 0x59, 0x13, 0xaa, 0x9d, 0xd0, 0x5f, 0xcc, 0x82, 0xe7, 0x72, 0x84, 0xbe,
-	0x6b, 0x35, 0xad, 0x96, 0x23, 0xb6, 0x21, 0x5d, 0x71, 0xe1, 0xcd, 0xf0, 0xe5, 0x42, 0x06, 0x6a,
-	0x31, 0x73, 0x4b, 0x49, 0xc5, 0x16, 0xc4, 0xbf, 0x58, 0xe0, 0x3c, 0x8b, 0xe4, 0x0c, 0xa9, 0xe3,
-	0x01, 0x54, 0x44, 0xb8, 0xda, 0x6e, 0x97, 0xc5, 0xec, 0x01, 0xd4, 0xfa, 0xc1, 0x12, 0xa3, 0x18,
-	0x7b, 0x81, 0x1c, 0xf9, 0x38, 0xa1, 0x76, 0x15, 0x51, 0x40, 0xd9, 0x7d, 0x70, 0x3a, 0x72, 0xfc,
-	0x16, 0x2f, 0xd6, 0x73, 0x74, 0xcb, 0xd4, 0x24, 0x07, 0xb2, 0xec, 0xd0, 0x7b, 0x8f, 0xae, 0xdd,
-	0xb4, 0x5a, 0x7b, 0x22, 0x07, 0x8a, 0x7a, 0x77, 0xae, 0xea, 0xe5, 0x50, 0xeb, 0xcf, 0xe6, 0x61,
-	0xa4, 0x04, 0xc6, 0xf3, 0x30, 0x88, 0x91, 0xd5, 0xa1, 0xdc, 0x8b, 0x22, 0x23, 0x57, 0x1f, 0xf9,
-	0x07, 0xa8, 0x9f, 0xfa, 0xe1, 0x78, 0xda, 0x95, 0x4a, 0x0a, 0x7c, 0xb7, 0xc0, 0x58, 0xb1, 0x3b,
-	0xb0, 0x43, 0xc6, 0x99, 0xba, 0x24, 0xd0, 0x28, 0x0d, 0x6f, 0x9c, 0x49, 0x02, 0x8d, 0xd2, 0x7d,
-	0x52, 0x6f, 0x8b, 0x24, 0xd0, 0xe8, 0xd0, 0xf7, 0xc6, 0x89, 0x6a, 0x5b, 0x24, 0x01, 0x63, 0x60,
-	0xbf, 0xf2, 0x70, 0x65, 0xa4, 0xd2, 0x99, 0xf7, 0x61, 0x7f, 0x8b, 0xdf, 0xc8, 0xbc, 0x0b, 0xbb,
-	0x22, 0x5c, 0xf5, 0xbb, 0xb1, 0x6b, 0x35, 0xcb, 0x2d, 0x5b, 0x98, 0x88, 0x0c, 0xa1, 0x2f, 0xa6,
-	0x53, 0x25, 0x4a, 0xe5, 0x00, 0xbf, 0x07, 0x3b, 0xe4, 0x8e, 0x9e, 0x32, 0xbf, 0xab, 0x8f, 0xfc,
-	0xb3, 0x05, 0xfb, 0x03, 0x79, 0x49, 0x32, 0xe2, 0x8c, 0xe6, 0x0c, 0x9c, 0x0c, 0xa4, 0xea, 0xea,
-	0xf1, 0x51, 0x3b, 0x5d, 0x9f, 0xf6, 0x95, 0xfa, 0x1c, 0xe9, 0x05, 0x2a, 0x5a, 0x8b, 0xfc, 0xf2,
-	0xc1, 0x53, 0xa8, 0xfd, 0x9a, 0xd4, 0x1a, 0xa6, 0xb8, 0x4e, 0x9d, 0x9e, 0xe2, 0x5a, 0x7b, 0xb2,
-	0x94, 0xfe, 0x22, 0xf1, 0xcf, 0x16, 0x49, 0xf0, 0xa4, 0xf4, 0xd8, 0xe2, 0x27, 0xc0, 0x3a, 0x11,
-	0x4a, 0x85, 0xd4, 0x60, 0x80, 0x71, 0x2c, 0xdf, 0xe0, 0xf5, 0x5f, 0x21, 0x71, 0xb6, 0xb4, 0xe5,
-	0x2c, 0x3f, 0x02, 0xd6, 0x45, 0x1f, 0x15, 0x9a, 0x85, 0xff, 0x43, 0x07, 0x3e, 0x4c, 0xd9, 0x6e,
-	0xae, 0x65, 0x87, 0x60, 0xeb, 0x5d, 0x27, 0xb2, 0xea, 0xf1, 0xed, 0xdc, 0x9c, 0xec, 0x61, 0x09,
-	0x2a, 0xe0, 0x5e, 0xda, 0xd4, 0xbc, 0x8f, 0x1b, 0x46, 0xf8, 0xcd, 0x22, 0xa5, 0x54, 0xe5, 0x22,
-	0x55, 0xf6, 0xe2, 0x0c, 0xd5, 0x49, 0x3a, 0xeb, 0xbf, 0x52, 0xf1, 0xae, 0x41, 0xf5, 0x42, 0x9e,
-	0xeb, 0x6c, 0x72, 0x87, 0xce, 0xd7, 0x8f, 0x5c, 0xd4, 0xf1, 0xd1, 0x32, 0x94, 0x7f, 0xd7, 0xa6,
-	0xe0, 0x9c, 0xfe, 0x8d, 0xa4, 0xab, 0x63, 0xde, 0x50, 0x16, 0xb3, 0x43, 0xd8, 0x25, 0xd6, 0xd8,
-	0xb5, 0x69, 0x3b, 0x6f, 0x15, 0xd4, 0x08, 0x93, 0xe6, 0xaf, 0xc1, 0x39, 0x0f, 0x27, 0x38, 0x54,
-	0x52, 0xd1, 0x54, 0x67, 0x61, 0xac, 0x52, 0x39, 0xfa, 0x4c, 0x6b, 0xa3, 0x93, 0xa9, 0x11, 0x49,
-	0xe5, 0x43, 0xf8, 0x9f, 0xe4, 0x60, 0xec, 0x96, 0x8b, 0x04, 0x94, 0x10, 0x69, 0xfe, 0xb4, 0xfe,
-	0x75, 0xd3, 0xb0, 0xbe, 0x6d, 0x1a, 0xd6, 0xf7, 0x4d, 0xc3, 0xfa, 0xf4, 0xa3, 0xf1, 0xdf, 0x68,
-	0x97, 0xfe, 0xb7, 0x8f, 0x7e, 0x06, 0x00, 0x00, 0xff, 0xff, 0xc6, 0x68, 0xdc, 0x63, 0x80, 0x05,
-	0x00, 0x00,
+	// 617 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x40,
+	0x10, 0xc5, 0x89, 0x53, 0x9a, 0xa9, 0x5a, 0xda, 0xa5, 0x42, 0xa6, 0x42, 0x51, 0xb4, 0x07, 0x5a,
+	0x7a, 0xe8, 0xa1, 0x5c, 0x10, 0x70, 0xa8, 0x9a, 0x04, 0x35, 0x12, 0x29, 0x62, 0x53, 0x71, 0xdf,
+	0x24, 0x23, 0xb0, 0xe2, 0xd8, 0xc1, 0xbb, 0x4e, 0x1a, 0x0e, 0xdc, 0xf9, 0x03, 0x24, 0xbe, 0x81,
+	0xff, 0xe0, 0xc8, 0x27, 0xa0, 0xf0, 0x23, 0x68, 0xc7, 0x6b, 0x3b, 0xb8, 0x94, 0x0a, 0x6e, 0x3b,
+	0x6f, 0x66, 0xe7, 0xbd, 0x7d, 0x9e, 0x31, 0x6c, 0x4e, 0x63, 0x7f, 0x26, 0x35, 0x1e, 0x4d, 0xe3,
+	0x48, 0x47, 0x6c, 0xdd, 0x0f, 0x35, 0xc6, 0xa1, 0x0c, 0xf8, 0x2b, 0xa8, 0x77, 0xc3, 0x11, 0x5e,
+	0xf6, 0x50, 0x4b, 0xd6, 0x84, 0x8d, 0x56, 0x14, 0x24, 0x93, 0xf0, 0xa5, 0x1c, 0x60, 0xe0, 0x39,
+	0x4d, 0xe7, 0xa0, 0x2e, 0x56, 0x21, 0x53, 0x71, 0xe1, 0x4f, 0xf0, 0x75, 0x22, 0x43, 0x9d, 0x4c,
+	0xbc, 0x4a, 0x5a, 0xb1, 0x02, 0xf1, 0xaf, 0x0e, 0xd4, 0x5f, 0xc4, 0x72, 0x82, 0xd4, 0x71, 0x0f,
+	0xd6, 0x45, 0x34, 0x5f, 0x6d, 0x97, 0xc7, 0xec, 0x21, 0x6c, 0x75, 0xc3, 0x19, 0xc6, 0x0a, 0x3b,
+	0xa1, 0x1c, 0x04, 0x38, 0xa2, 0x76, 0xeb, 0xa2, 0x84, 0xb2, 0x07, 0x50, 0x6f, 0xc9, 0xe1, 0x3b,
+	0xbc, 0x58, 0x4c, 0xd1, 0xab, 0x52, 0x93, 0x02, 0xc8, 0xb3, 0x7d, 0xff, 0x03, 0x7a, 0x6e, 0xd3,
+	0x39, 0xd8, 0x14, 0x05, 0x50, 0xd6, 0x5b, 0xbb, 0xaa, 0x97, 0xc3, 0x56, 0x77, 0x32, 0x8d, 0x62,
+	0x2d, 0x50, 0x4d, 0xa3, 0x50, 0x21, 0xdb, 0x86, 0x6a, 0x27, 0x8e, 0xad, 0x5c, 0x73, 0xe4, 0x1f,
+	0x61, 0xfb, 0x34, 0x88, 0x86, 0xe3, 0xb6, 0xd4, 0x52, 0xe0, 0xfb, 0x04, 0x95, 0x66, 0xbb, 0x50,
+	0x23, 0xe3, 0x6c, 0x5d, 0x1a, 0x18, 0x94, 0x1e, 0x6f, 0x9d, 0x49, 0x03, 0x83, 0xd2, 0x7d, 0x52,
+	0xef, 0x8a, 0x34, 0x30, 0x68, 0x3f, 0xf0, 0x87, 0xa9, 0x6a, 0x57, 0xa4, 0x01, 0x63, 0xe0, 0xbe,
+	0xf1, 0x71, 0x6e, 0xa5, 0xd2, 0x99, 0x77, 0x61, 0x67, 0x85, 0xdf, 0xca, 0xbc, 0x07, 0x6b, 0x22,
+	0x9a, 0x77, 0xdb, 0xca, 0x73, 0x9a, 0xd5, 0x03, 0x57, 0xd8, 0x88, 0x0c, 0xa1, 0x2f, 0x66, 0x52,
+	0x15, 0x4a, 0x15, 0x00, 0xbf, 0x0f, 0x35, 0x72, 0xc7, 0xbc, 0xb2, 0xb8, 0x6b, 0x8e, 0xfc, 0x8b,
+	0x03, 0x3b, 0x3d, 0x79, 0x49, 0x32, 0x54, 0x4e, 0x73, 0x06, 0xf5, 0x1c, 0xa4, 0xea, 0x8d, 0xe3,
+	0xc3, 0xa3, 0x6c, 0x7c, 0x8e, 0xae, 0xd4, 0x17, 0x48, 0x27, 0xd4, 0xf1, 0x42, 0x14, 0x97, 0xf7,
+	0x9e, 0xc3, 0xd6, 0xef, 0x49, 0xa3, 0x61, 0x8c, 0x8b, 0xcc, 0xe9, 0x31, 0x2e, 0x8c, 0x27, 0x33,
+	0x19, 0x24, 0xa9, 0x7f, 0xae, 0x48, 0x83, 0xa7, 0x95, 0x27, 0x0e, 0x3f, 0x01, 0xd6, 0x8a, 0x51,
+	0x6a, 0xa4, 0x06, 0x3d, 0x54, 0x4a, 0xbe, 0xc5, 0xeb, 0xbf, 0x42, 0xea, 0x6c, 0x65, 0xc5, 0x59,
+	0x7e, 0x08, 0xac, 0x8d, 0x01, 0x6a, 0xb4, 0x03, 0xff, 0x97, 0x0e, 0xbc, 0x9f, 0xb1, 0xdd, 0x5c,
+	0xcb, 0xf6, 0xc1, 0x35, 0xb3, 0x4e, 0x64, 0x1b, 0xc7, 0x77, 0x0b, 0x73, 0xf2, 0xc5, 0x12, 0x54,
+	0xc0, 0xfd, 0xac, 0xa9, 0xdd, 0x8f, 0x1b, 0x9e, 0xf0, 0x87, 0x41, 0xca, 0xa8, 0xaa, 0x65, 0xaa,
+	0x7c, 0xe3, 0x2c, 0xd5, 0x49, 0xf6, 0xd6, 0xff, 0xa5, 0xe2, 0x6d, 0x8b, 0x9a, 0x81, 0x3c, 0x37,
+	0xd9, 0xf4, 0x0e, 0x9d, 0xaf, 0x7f, 0x72, 0x59, 0xc7, 0x27, 0xc7, 0x52, 0xfe, 0x5b, 0x9b, 0x92,
+	0x73, 0xe6, 0x37, 0x92, 0x8d, 0x8e, 0xdd, 0xa1, 0x3c, 0x66, 0xfb, 0xb0, 0x46, 0xac, 0xca, 0x73,
+	0x69, 0x3a, 0xef, 0x94, 0xd4, 0x08, 0x9b, 0xe6, 0x12, 0xe0, 0x3c, 0x1a, 0x61, 0x5f, 0x4b, 0x9d,
+	0x28, 0xa3, 0xe7, 0x2c, 0x52, 0x3a, 0xd3, 0x63, 0xce, 0x34, 0x37, 0x5a, 0xea, 0xdc, 0x09, 0x0a,
+	0xd8, 0x23, 0xb8, 0x4d, 0x7a, 0x50, 0x79, 0xd5, 0x32, 0x03, 0x25, 0x44, 0x96, 0xe7, 0xcf, 0x60,
+	0xb3, 0x15, 0x24, 0x4a, 0x63, 0x6c, 0x59, 0x0e, 0xa1, 0x66, 0x38, 0xb3, 0xcd, 0xd9, 0x2d, 0x6e,
+	0x16, 0x52, 0x44, 0x5a, 0x72, 0xba, 0xfd, 0x6d, 0xd9, 0x70, 0xbe, 0x2f, 0x1b, 0xce, 0x8f, 0x65,
+	0xc3, 0xf9, 0xfc, 0xb3, 0x71, 0x6b, 0xb0, 0x46, 0x7f, 0xeb, 0xc7, 0xbf, 0x02, 0x00, 0x00, 0xff,
+	0xff, 0x77, 0x02, 0x29, 0xbb, 0xbe, 0x05, 0x00, 0x00,
 }
diff --git a/internal/private.proto b/internal/private.proto
index b791efb6c..b83683d51 100644
--- a/internal/private.proto
+++ b/internal/private.proto
@@ -77,8 +77,12 @@ message Index {
     repeated Frame Frames = 4;
 }
 
-message NodeState {
+message NodeStatus {
     string Host = 1;
     string State = 2;
     repeated Index Indexes = 3;
 }
+
+message ClusterStatus {
+    repeated NodeStatus Nodes = 1;
+}
diff --git a/server.go b/server.go
index 6aef3dd62..336d76b54 100644
--- a/server.go
+++ b/server.go
@@ -118,6 +118,7 @@ func (s *Server) Open() error {
 
 	// Initialize HTTP handler.
 	s.Handler.Broadcaster = s.Broadcaster
+	s.Handler.StatusHandler = s
 	s.Handler.Host = s.Host
 	s.Handler.Cluster = s.Cluster
 	s.Handler.Executor = e
@@ -271,28 +272,49 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
 	return nil
 }
 
-// Server implements gossip.StateHandler.
-// LocalState returns the state of the local node as well as the
+// Server implements StatusHandler.
+// LocalStatus returns the state of the local node as well as the
 // holder (indexes/frames) according to the local node.
 // In a gossip implementation, memberlist.Delegate.LocalState() uses this.
-func (s *Server) LocalState() (proto.Message, error) {
+func (s *Server) LocalStatus() (proto.Message, error) {
 	if s.Holder == nil {
 		return nil, errors.New("Server.Holder is nil.")
 	}
-	return &internal.NodeState{
+	return &internal.NodeStatus{
 		Host:    s.Host,
-		State:   "OK", // TODO: make this work, pull from s.Cluster.Node
+		State:   NodeStateUp,
 		Indexes: encodeIndexes(s.Holder.Indexes()),
 	}, nil
 }
 
-// HandleRemoteState receives incoming NodeState from remote nodes.
-func (s *Server) HandleRemoteState(pb proto.Message) error {
-	return s.mergeRemoteState(pb.(*internal.NodeState))
+// ClusterStatus returns the NodeState for all nodes in the cluster.
+func (s *Server) ClusterStatus() (proto.Message, error) {
+	// Update local Node.state.
+	ns, err := s.LocalStatus()
+	if err != nil {
+		return nil, err
+	}
+	node := s.Cluster.NodeByHost(s.Host)
+	node.SetStatus(ns.(*internal.NodeStatus))
+
+	// Update NodeState for all nodes.
+	for host, nodeState := range s.Cluster.NodeStates() {
+		node := s.Cluster.NodeByHost(host)
+		node.SetState(nodeState)
+	}
+
+	return s.Cluster.Status(), nil
 }
 
-func (s *Server) mergeRemoteState(ns *internal.NodeState) error {
-	// TODO: update some node state value in the cluster (it should be in cluster.node i guess)
+// HandleRemoteStatus receives incoming NodeState from remote nodes.
+func (s *Server) HandleRemoteStatus(pb proto.Message) error {
+	return s.mergeRemoteStatus(pb.(*internal.NodeStatus))
+}
+
+func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
+	// Update Node.state.
+	node := s.Cluster.NodeByHost(ns.Host)
+	node.SetStatus(ns)
 
 	// Create indexes that don't exist.
 	for _, index := range ns.Indexes {
@@ -364,3 +386,12 @@ func checkMaxSlices(hostport string) (map[string]uint64, error) {
 
 	return pb.MaxSlices, nil
 }
+
+// StatusHandler specifies two methods which an object must implement to share
+// state in the cluster. These are used by the GossipNodeSet to implement the
+// LocalState and MergeRemoteState methods of memberlist.Delegate
+type StatusHandler interface {
+	LocalStatus() (proto.Message, error)
+	ClusterStatus() (proto.Message, error)
+	HandleRemoteStatus(proto.Message) error
+}

From 5857438d8bbd44c6604c065b05f14595a9d20e7c Mon Sep 17 00:00:00 2001
From: Michael Baird <mbaird@umbel.com>
Date: Mon, 24 Apr 2017 13:56:26 -0500
Subject: [PATCH 2/5] fixed state to status

---
 cluster.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cluster.go b/cluster.go
index f9f22464f..0ed3dfcb2 100644
--- a/cluster.go
+++ b/cluster.go
@@ -24,7 +24,7 @@ type Node struct {
 	Host         string `json:"host"`
 	InternalHost string `json:"internalHost"`
 
-	status *internal.NodeStatus `json:"state"`
+	status *internal.NodeStatus `json:"status"`
 }
 
 // SetStatus sets the NodeStatus.

From acc4b022d78cb77881996ba0081355d5aaef1db8 Mon Sep 17 00:00:00 2001
From: Michael Baird <mbaird@umbel.com>
Date: Mon, 24 Apr 2017 13:57:35 -0500
Subject: [PATCH 3/5] added Join method to Static Nodeset so that Status is
 calculated with the local node

---
 broadcast.go     | 5 +++++
 server/server.go | 1 +
 2 files changed, 6 insertions(+)

diff --git a/broadcast.go b/broadcast.go
index 5a1600fbb..9e04d04f6 100644
--- a/broadcast.go
+++ b/broadcast.go
@@ -34,6 +34,11 @@ func (s *StaticNodeSet) Open() error {
 	return nil
 }
 
+func (s *StaticNodeSet) Join(nodes []*Node) error {
+	s.nodes = nodes
+	return nil
+}
+
 // Broadcaster is an interface for broadcasting messages.
 type Broadcaster interface {
 	SendSync(pb proto.Message) error
diff --git a/server/server.go b/server/server.go
index 153f2dc23..d5983d7c3 100644
--- a/server/server.go
+++ b/server/server.go
@@ -163,6 +163,7 @@ func (m *Command) SetupServer() error {
 		m.Server.Broadcaster = pilosa.NopBroadcaster
 		m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet()
 		m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
+		m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes)
 	default:
 		return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type)
 	}

From 1dc5d6f3485537f75223e609659e6efb42e43eda Mon Sep 17 00:00:00 2001
From: Michael Baird <mbaird@umbel.com>
Date: Mon, 24 Apr 2017 14:14:16 -0500
Subject: [PATCH 4/5] handle Join error

---
 server/server.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/server/server.go b/server/server.go
index d5983d7c3..da5fe9a8e 100644
--- a/server/server.go
+++ b/server/server.go
@@ -163,7 +163,10 @@ func (m *Command) SetupServer() error {
 		m.Server.Broadcaster = pilosa.NopBroadcaster
 		m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet()
 		m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
-		m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes)
+		err := m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes)
+		if err != nil {
+			return err
+		}
 	default:
 		return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type)
 	}

From 445023a04f177c7706a5e958e4556cd22b003435 Mon Sep 17 00:00:00 2001
From: Michael Baird <mbaird@umbel.com>
Date: Mon, 24 Apr 2017 14:21:50 -0500
Subject: [PATCH 5/5] fixed some comments

---
 server/server.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/server/server.go b/server/server.go
index da5fe9a8e..b6087ba21 100644
--- a/server/server.go
+++ b/server/server.go
@@ -1,4 +1,4 @@
-// package server contains the `pilosa server` subcommand which runs Pilosa
+// Package server contains the `pilosa server` subcommand which runs Pilosa
 // itself. The purpose of this package is to define an easily tested Command
 // object which handles interpreting configuration and setting up all the
 // objects that Pilosa needs.
@@ -50,7 +50,7 @@ type Command struct {
 	Done chan struct{}
 }
 
-// NewMain returns a new instance of Main.
+// NewCommand returns a new instance of Main.
 func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
 	return &Command{
 		Server: pilosa.NewServer(),
@@ -89,6 +89,7 @@ func (m *Command) Run(args ...string) (err error) {
 	return nil
 }
 
+// SetupServer use the cluster configuration to setup this server
 func (m *Command) SetupServer() error {
 	cluster := pilosa.NewCluster()
 	cluster.ReplicaN = m.Config.Cluster.ReplicaN
@@ -168,7 +169,7 @@ func (m *Command) SetupServer() error {
 			return err
 		}
 	default:
-		return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type)
+		return fmt.Errorf("'%v' is not a supported value for broadcaster type", m.Config.Cluster.Type)
 	}
 
 	// Set configuration options.