diff --git a/docs/configuration.md b/docs/configuration.md index 0798d1381..6c20e6ec5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -111,7 +111,7 @@ Any flag that has a value that is a comma separated list on the command line bec #### Gossip Seed -* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`. +* Description: When using the gossip [Cluster Type]({{< ref "#cluster-type" >}}), this specifies which internal host(s) should be used to initialize membership in the cluster. Typcially this can be the address of any available host in the cluster. You may enter multiple seeds by separating them with a comma. For example, when starting a three-node cluster made up of `node0`, `node1`, and `node2`, the `gossip-seed` for all three nodes can be configured to be the address of `node0`. * Flag: `--gossip.seed="localhost:11101"` * Env: `PILOSA_GOSSIP_SEED="localhost:11101"` * Config: diff --git a/gossip/gossip.go b/gossip/gossip.go index acfad66a1..a61a5b9b7 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -92,13 +92,19 @@ func (g *GossipMemberSet) Open(n *pilosa.Node) error { RetransmitMult: 3, } - uri, err := pilosa.NewURIFromAddress(g.config.gossipSeed) - if err != nil { - return fmt.Errorf("new uri from address: %s", err) + parts := strings.Split(g.config.gossipSeed, ",") + var uris = make([]*pilosa.URI, len(parts)) + for i, addr := range parts { + uris[i], err = pilosa.NewURIFromAddress(addr) + if err != nil { + return fmt.Errorf("new uri from address: %s", err) + } } - // attach to gossip seed node - nodes := []*pilosa.Node{&pilosa.Node{URI: *uri}} //TODO: support a list of seeds + var nodes = make([]*pilosa.Node, len(uris)) + for i, uri := range uris { + nodes[i] = &pilosa.Node{URI: *uri} + } g.mu.RLock() err = g.joinWithRetry(pilosa.URIs(pilosa.Nodes(nodes).URIs()).HostPortStrings()) diff --git a/server/cluster_test.go b/server/cluster_test.go index 23284bb00..82c54bb96 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -435,3 +435,64 @@ func TestClusterResize_AddNode(t *testing.T) { } }) } + +// Ensure that redundant gossip seeds are used +func TestCluster_GossipMembership(t *testing.T) { + t.Run("Node0Down", func(t *testing.T) { + // Configure node0 + m0 := test.NewMainWithCluster() + defer m0.Close() + + seed, coord, err := m0.RunWithTransport("localhost", 0, "", pilosa.URI{}) + if err != nil { + t.Fatal(err) + } + + // Configure node1 + m1 := test.NewMainWithCluster() + defer m1.Close() + + var eg errgroup.Group + eg.Go(func() error { + // Pass invalid seed as first in list + _, _, err = m1.RunWithTransport("localhost", 0, "http://localhost:8765,"+seed, coord) + if err != nil { + return err + } + return nil + }) + + // Configure node2 + m2 := test.NewMainWithCluster() + defer m2.Close() + + eg.Go(func() error { + // Pass invalid seed as last in list + _, _, err = m2.RunWithTransport("localhost", 0, seed+",http://localhost:8765", coord) + if err != nil { + return err + } + return nil + }) + + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // Give the cluster time to settle. + time.Sleep(1 * time.Second) + + if m0.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node0 cluster state: %s", m0.Server.Cluster.State()) + } else if m1.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node1 cluster state: %s", m1.Server.Cluster.State()) + } else if m2.Server.Cluster.State() != pilosa.ClusterStateNormal { + t.Fatalf("unexpected node2 cluster state: %s", m2.Server.Cluster.State()) + } + + numNodes := len(m0.Server.Cluster.Status().Nodes) + if numNodes != 3 { + t.Fatalf("Expected 3 nodes, got %d", numNodes) + } + }) +}