diff --git a/api.go b/api.go index 61a6a7060..92a1bc1b7 100644 --- a/api.go +++ b/api.go @@ -118,8 +118,7 @@ func (api *API) SetAPIOptions(opts ...apiOption) error { // validAPIMethods specifies the api methods that are valid for each // cluster state. var validAPIMethods = map[disco.ClusterState]map[apiMethod]struct{}{ - disco.ClusterStateStarting: methodsCommon, - disco.ClusterStateNormal: appendMap(methodsCommon, methodsNormal), + disco.ClusterStateNormal: appendMap(methodsCommon, methodsNormal), // Ideally, this would be just `appendMap(methodsCommon, methodsDegraded)`, // but in an attempt to reduce the influence that state (determined by etcd) // has on a node under load, this is set to effectively allow all requests diff --git a/disco/disco.go b/disco/disco.go index 66daba6aa..6761858f4 100644 --- a/disco/disco.go +++ b/disco/disco.go @@ -54,7 +54,6 @@ const ( InitialClusterStateExisting InitialClusterState = "existing" ClusterStateUnknown ClusterState = "UNKNOWN" // default cluster state. It is returned when we are not able to get the real actual state. - ClusterStateStarting ClusterState = "STARTING" // cluster is starting and some internal services are not ready yet. ClusterStateDegraded ClusterState = "DEGRADED" // cluster is running but we've lost some # of hosts >0 but < replicaN. Only read queries are allowed. ClusterStateNormal ClusterState = "NORMAL" // cluster is up and running. ClusterStateDown ClusterState = "DOWN" // cluster is unable to serve queries. diff --git a/etcd/embed.go b/etcd/embed.go index 1ec887bcd..8f6032c76 100644 --- a/etcd/embed.go +++ b/etcd/embed.go @@ -490,7 +490,6 @@ func (e *Etcd) ClusterState(ctx context.Context) (out disco.ClusterState, err er } var ( heartbeats int = 0 - starting bool ) e.nodeMu.Lock() nodes := e.populateNodeStates(ctx) @@ -500,28 +499,16 @@ func (e *Etcd) ClusterState(ctx context.Context) (out disco.ClusterState, err er return disco.ClusterStateUnknown, err } for _, node := range nodes { - switch node.State { - case disco.NodeStateStarting: - starting = true - case disco.NodeStateUnknown: - continue + if node.State == disco.NodeStateStarted { + heartbeats++ } - - heartbeats++ } - - if starting { - return disco.ClusterStateStarting, nil - } - if heartbeats < len(e.knownNodes) { if len(e.knownNodes)-heartbeats >= e.replicas { return disco.ClusterStateDown, nil } - return disco.ClusterStateDegraded, nil } - return disco.ClusterStateNormal, nil } diff --git a/etcd/leasedkv_test.go b/etcd/leasedkv_test.go index dd9b12e0a..f1c127bb6 100644 --- a/etcd/leasedkv_test.go +++ b/etcd/leasedkv_test.go @@ -29,26 +29,23 @@ func TestClusterKv(t *testing.T) { if err != nil { t.Fatalf("starting cluster: %v", err) } - c.MustAwaitClusterState(disco.ClusterStateStarting, 10*time.Second) + c.MustAwaitClusterState(disco.ClusterStateDown, 10*time.Second) err = c.BringUp() if err != nil { t.Fatalf("bringing up cluster: %v", err) } c.MustAwaitClusterState(disco.ClusterStateNormal, 10*time.Second) - _, err = c.Elect() - if err != nil { - t.Fatalf("trying to cause election: %v", err) - } ctx := context.TODO() c.nodes[0].SetState(ctx, disco.NodeStateStarting) c.nodes[1].SetState(ctx, disco.NodeStateStarting) - c.MustAwaitClusterState(disco.ClusterStateStarting, 10*time.Second) + c.MustAwaitClusterState(disco.ClusterStateDown, 10*time.Second) c.nodes[0].SetState(ctx, disco.NodeStateStarted) - c.nodes[1].SetState(ctx, disco.NodeStateStarted) // Two of three nodes are up, one is down, we have 2 replicas, so // we should be able to handle reads but not writes, so we're in // a Degraded state. c.MustAwaitClusterState(disco.ClusterStateDegraded, 10*time.Second) + c.nodes[1].SetState(ctx, disco.NodeStateStarted) + c.MustAwaitClusterState(disco.ClusterStateNormal, 10*time.Second) err = c.Stop() if err != nil { t.Fatalf("stopping cluster: %v", err) diff --git a/server.go b/server.go index 337b505b0..0f3204bea 100644 --- a/server.go +++ b/server.go @@ -694,29 +694,29 @@ func (s *Server) Open() error { if !timer.Stop() { <-timer.C } + // wait for cluster to achieve Normal state + // + // This used to loop as long as the cluster was Starting, Down, + // or Unknown. It would come up in a Degraded state, except + // that during startup, as long as at least one node was Starting, + // we'd stay in Starting rather than Degraded. We've dropped the + // special case of Starting state, so now we just want to wait + // for Normal. for { state, err := s.noder.ClusterState(ctx) if err != nil { s.logger.Printf("failed to check cluster state: %v", err) - timer.Reset(time.Second) - select { - case <-s.closing: - return - case <-timer.C: - continue - } } - switch state { - case disco.ClusterStateStarting, disco.ClusterStateUnknown, disco.ClusterStateDown: - timer.Reset(time.Second) - select { - case <-s.closing: - return - case <-timer.C: - continue - } + if state == disco.ClusterStateNormal { + break + } + timer.Reset(time.Second) + select { + case <-s.closing: + return + case <-timer.C: + continue } - break } start := time.Now()