Add support for comma-separated list of gossip seeds for redundancy.

docs/configuration.md

@@ -111,7 +111,7 @@ Any flag that has a value that is a comma separated list on the command line bec
 
 #### Gossip Seed
 
-* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`.
+* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host(s) should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. You may enter multiple seeds by separating them with a comma. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`.
 * Flag: `--gossip.seed="localhost:11101"`
 * Env: `PILOSA_GOSSIP_SEED="localhost:11101"`
 * Config:

gossip/gossip.go

@@ -92,13 +92,19 @@ func (g *GossipMemberSet) Open(n *pilosa.Node) error {
 		RetransmitMult: 3,
 	}
 
-	uri, err := pilosa.NewURIFromAddress(g.config.gossipSeed)
-	if err != nil {
-		return fmt.Errorf("new uri from address: %s", err)
+	parts := strings.Split(g.config.gossipSeed, ",")
+	var uris = make([]*pilosa.URI, len(parts))
+	for i, addr := range parts {
+		uris[i], err = pilosa.NewURIFromAddress(addr)
+		if err != nil {
+			return fmt.Errorf("new uri from address: %s", err)
+		}
 	}
 
-	// attach to gossip seed node
-	nodes := []*pilosa.Node{&pilosa.Node{URI: *uri}} //TODO: support a list of seeds
+	var nodes = make([]*pilosa.Node, len(uris))
+	for i, uri := range uris {
+		nodes[i] = &pilosa.Node{URI: *uri}
+	}
 
 	g.mu.RLock()
 	err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings())

server/cluster_test.go

@@ -435,3 +435,64 @@ func TestClusterResize_AddNode(t *testing.T) {
 		}
 	})
 }
+
+// Ensure that redundant gossip seeds are used
+func TestCluster_GossipMembership(t *testing.T) {
+	t.Run("Node0Down", func(t *testing.T) {
+		// Configure node0
+		m0 := test.NewMainWithCluster()
+		defer m0.Close()
+
+		seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Configure node1
+		m1 := test.NewMainWithCluster()
+		defer m1.Close()
+
+		var eg errgroup.Group
+		eg.Go(func() error {
+			// Pass invalid seed as first in list
+			_, _, err = m1.RunWithTransport("localhost", 0, "http://localhost:8765,"+seed, coord)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
+		// Configure node2
+		m2 := test.NewMainWithCluster()
+		defer m2.Close()
+
+		eg.Go(func() error {
+			// Pass invalid seed as last in list
+			_, _, err = m2.RunWithTransport("localhost", 0, seed+",http://localhost:8765", coord)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
+		if err := eg.Wait(); err != nil {
+			t.Fatal(err)
+		}
+
+		// Give the cluster time to settle.
+		time.Sleep(1 * time.Second)
+
+		if m0.Server.Cluster.State() != pilosa.ClusterStateNormal {
+			t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State())
+		} else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal {
+			t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State())
+		} else if m2.Server.Cluster.State() != pilosa.ClusterStateNormal {
+			t.Fatalf("unexpected node2 cluster state: %s", m2.Server.Cluster.State())
+		}
+
+		numNodes := len(m0.Server.Cluster.Status().Nodes)
+		if numNodes != 3 {
+			t.Fatalf("Expected 3 nodes, got %d", numNodes)
+		}
+	})
+}