From ccdd6262c59a8c3ad107fea2fc337956044ef4c3 Mon Sep 17 00:00:00 2001
From: Travis Turner <travis@pilosa.com>
Date: Tue, 28 Nov 2017 12:43:49 -0600
Subject: [PATCH] add error logging for non-topology nodeJoin

---
 cluster.go       |  4 +++-
 gossip/gossip.go |  4 ++--
 server.go        | 21 ++++-----------------
 3 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/cluster.go b/cluster.go
index 9f30c0f94..6d02d352b 100644
--- a/cluster.go
+++ b/cluster.go
@@ -1382,7 +1382,9 @@ func (c *Cluster) nodeJoin(uri URI) error {
 	if c.needTopologyAgreement() {
 		// A host that is not part of the topology can't be added to the STARTING cluster.
 		if !c.Topology.ContainsURI(uri) {
-			return fmt.Errorf("host is not in topology: %v", uri)
+			err := fmt.Sprintf("host is not in topology: %v", uri)
+			c.logger().Print(err)
+			return errors.New(err)
 		}
 
 		if err := c.AddNode(uri); err != nil {
diff --git a/gossip/gossip.go b/gossip/gossip.go
index 2c5c85faf..7e89e17c1 100644
--- a/gossip/gossip.go
+++ b/gossip/gossip.go
@@ -148,7 +148,7 @@ func NewGossipMemberSet(name string, gossipHost string, gossipPort int, gossipSe
 	g.config.memberlistConfig.BindPort = gossipPort
 	g.config.memberlistConfig.AdvertiseAddr = pilosa.HostToIP(gossipHost)
 	g.config.memberlistConfig.AdvertisePort = gossipPort
-	//g.config.memberlistConfig.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig. // TODO travis: change this from 0
+	//g.config.memberlistConfig.PushPullInterval = 0 * time.Second // Default is 15s in DefaultLocalConfig.
 	g.config.memberlistConfig.Delegate = g
 	g.config.memberlistConfig.SecretKey = secretKey
 	g.config.memberlistConfig.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate)
@@ -336,7 +336,7 @@ func (g *GossipEventReceiver) listen() {
 			Event: nodeEventType,
 			URI:   *uri,
 		}
-		g.eventHandler.ReceiveEvent(ne)
+		_ = g.eventHandler.ReceiveEvent(ne) // TODO: don't swallow this error
 	}
 }
 
diff --git a/server.go b/server.go
index 7328eac80..8a4ad38d5 100644
--- a/server.go
+++ b/server.go
@@ -432,6 +432,10 @@ func (s *Server) ClusterStatus() (proto.Message, error) {
 
 // HandleRemoteStatus receives incoming NodeStatus from remote nodes.
 func (s *Server) HandleRemoteStatus(pb proto.Message) error {
+	// Ignore NodeStatus messages until the cluster is in a Normal state.
+	if s.Cluster.State != ClusterStateNormal {
+		return nil
+	}
 	return s.mergeRemoteStatus(pb.(*internal.NodeStatus))
 }
 
@@ -441,23 +445,6 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
 		return nil
 	}
 
-	// If this node is still STARTING, don't apply remote status.
-	// There is an issue where starting up a cluster with existing
-	// data will error on `flock: resource temporarily unavailable`.
-	// This is because the ApplySchema creates/opens indexes before
-	// Holder.Open() has run. When Holder.Open() runs later, the
-	// fragment files are locked.
-	// TODO: There is still a race condition where the coordinator
-	// changes state to NORMAL, broadcasts that to the remote node,
-	// the remote node receives a `NodeStatus` (with schema) before
-	// running `Holder.Open()`. In that case, state would be NORMAL,
-	// meaning this check wouldn't pass, and `Holder.Open()` still
-	// hasn't run. We may need to track whether `Holder.Open()` has
-	// run, and use that to determine if we bail here.
-	if s.Cluster.State == ClusterStateStarting {
-		return nil
-	}
-
 	// Sync schema.
 	if err := s.Holder.ApplySchema(ns.Schema); err != nil {
 		return err