From 7e6c406212521d0204e6b31755be65ff85ab3bdd Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 10 Dec 2018 14:53:06 -0600 Subject: [PATCH 1/2] fix bug where cluster goes into RESIZING instead of NORMAL running "make clustertests DOCKER_COMPOSE=internal/clustertests/docker-compose-replication2.yml" shows this issue (just remove the change in cluster.go). Also removed two unrelated lines of code that appear to be doing absolutely nothing. --- Makefile | 14 +++-- cluster.go | 2 +- .../docker-compose-replication2.yml | 58 +++++++++++++++++++ 3 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 internal/clustertests/docker-compose-replication2.yml diff --git a/Makefile b/Makefile index af86b8241..8fa366137 100644 --- a/Makefile +++ b/Makefile @@ -68,20 +68,24 @@ release: check-clean $(MAKE) release-build GOOS=linux GOARCH=386 $(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1 + +# try (e.g.) internal/clustertests/docker-compose-replication2.yml +DOCKER_COMPOSE=internal/clustertests/docker-compose.yml + # Run cluster integration tests using docker. Requires docker daemon to be # running. This will catch changes to internal/clustertests/*.go, but if you # make changes to Pilosa, you'll want to run clustertests-build to rebuild the # pilosa image. clustertests: - docker-compose -f internal/clustertests/docker-compose.yml down - docker-compose -f internal/clustertests/docker-compose.yml build client1 - docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 + docker-compose -f $(DOCKER_COMPOSE) down + docker-compose -f $(DOCKER_COMPOSE) build client1 + docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 # Like clustertests, but rebuilds all images. clustertests-build: - docker-compose -f internal/clustertests/docker-compose.yml down - docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 --build + docker-compose -f $(DOCKER_COMPOSE) down + docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build # Create prerelease builds prerelease: vendor diff --git a/cluster.go b/cluster.go index 155c4a4aa..af54132e8 100644 --- a/cluster.go +++ b/cluster.go @@ -983,7 +983,7 @@ func (c *cluster) markAsJoined() { // needTopologyAgreement is unprotected. func (c *cluster) needTopologyAgreement() bool { - return c.state == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) + return (c.state == ClusterStateStarting || c.state == ClusterStateDegraded) && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) } // haveTopologyAgreement is unprotected. diff --git a/internal/clustertests/docker-compose-replication2.yml b/internal/clustertests/docker-compose-replication2.yml new file mode 100644 index 000000000..291699aee --- /dev/null +++ b/internal/clustertests/docker-compose-replication2.yml @@ -0,0 +1,58 @@ +version: '2' +services: + pilosa1: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33455:10101" + environment: + - PILOSA_CLUSTER_COORDINATOR=true + - PILOSA_GOSSIP_SEEDS=pilosa1:14000 + - PILOSA_CLUSTER_REPLICAS=2 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa1:10101" + pilosa2: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33456:10101" + environment: + - PILOSA_GOSSIP_SEEDS=pilosa1:14000 + - PILOSA_CLUSTER_REPLICAS=2 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa2:10101" + pilosa3: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33457:10101" + environment: + - PILOSA_GOSSIP_SEEDS=pilosa1:14000,pilosa2:14000 + - PILOSA_CLUSTER_REPLICAS=2 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa3:10101" + client1: + build: + context: . + environment: + - ENABLE_PILOSA_CLUSTER_TESTS=1 + networks: + - pilosanet + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: + - "go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests" +networks: + pilosanet: From 2c4401db8c72b3b6b2e88df7aada2bb1d5e1bc85 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 10 Dec 2018 15:48:25 -0600 Subject: [PATCH 2/2] hopefully fix data race --- gossip/gossip.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 8f19203f5..491e14cf2 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -333,15 +333,30 @@ func newEventReceiver(logger *log.Logger, papi *pilosa.API) *eventReceiver { } func (g *eventReceiver) NotifyJoin(n *memberlist.Node) { - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: n} + // copy node to avoid data race + n2 := *n + n2.Meta = make([]byte, len(n.Meta)) + copy(n2.Meta, n.Meta) + + g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2} } func (g *eventReceiver) NotifyLeave(n *memberlist.Node) { - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: n} + // copy node to avoid data race + n2 := *n + n2.Meta = make([]byte, len(n.Meta)) + copy(n2.Meta, n.Meta) + + g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2} } func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) { - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: n} + // copy node to avoid data race + n2 := *n + n2.Meta = make([]byte, len(n.Meta)) + copy(n2.Meta, n.Meta) + + g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2} } func (g *eventReceiver) listen() {