Merge pull request #1889 from molecula/1174-projectify-clustertests

add project support to clustertests to allow for concurrent runs

.gitlab/.gitlab-ci.yml

@@ -259,10 +259,11 @@ build arm container fb:
 # 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user
 # TODO: (I think) get clustertests coverage added to coverage report
 clustertests:
+  variables:
+    PROJECT: clustertests_${CI_CONCURRENT_ID}
   stage: integration
   tags:
     - shell
-    - gcp # this is to restrict to the GCP runner we set up manually, once all our runners are set up properly we can remove this.
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:

Dockerfile-clustertests

@@ -18,6 +18,10 @@ RUN chmod +x /pumba
 RUN apt update
 RUN apt install -y docker.io 
 
+# add docker-compose so tests can use it for stuff
+ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
+RUN chmod +x /usr/local/bin/docker-compose
+
 RUN cp /go/bin/featurebase /featurebase
 
 COPY NOTICE /NOTICE

Makefile

@@ -140,19 +140,23 @@ package:
 	nfpm package --packager deb --target featurebase_$(VERSION_ID).deb
 	nfpm package --packager rpm --target featurebase_$(VERSION_ID).rpm
 
-# try (e.g.) internal/clustertests/docker-compose-replication2.yml
-DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
+
+# We allow setting a custom docker-compose "project". Multiple of the
+# same docker-compose environment can exist simultaneously as long as
+# they use different projects (the project name is prepended to
+# container names and such). This is useful in a CI environment where
+# we might be running multiple instances of the tests concurrently.
+PROJECT ?= clustertests
+DOCKER_COMPOSE = docker-compose -p $(PROJECT)
 
 # Run cluster integration tests using docker. Requires docker daemon to be
-# running. This will catch changes to internal/clustertests/*.go, but if you
-# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
-# pilosa image.
+# running and docker-compose to be installed.
 clustertests: vendor
-	docker-compose -f $(DOCKER_COMPOSE) down
-	docker-compose -f $(DOCKER_COMPOSE) build
-	docker-compose -f $(DOCKER_COMPOSE) up -d pilosa1 pilosa2 pilosa3
-	docker-compose -f $(DOCKER_COMPOSE) run client1
-	docker-compose -f $(DOCKER_COMPOSE) down
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
+	PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
 
 # Install Pilosa

internal/clustertests/cluster_test.go

@@ -16,6 +16,20 @@ import (
 	picli "github.com/molecula/featurebase/v3/http"
 )
 
+// container turns a docker-compose service name into a container name
+// assuming the project name is set in the enviroment as PROJECT. This
+// refers to the "-p" argument to docker-compose. NOTE: this assumes
+// docker-compose joins the project name with a separating
+// underscore... this may not always be true as I've seen a dash used
+// as well, but I think it is true in recent versions.
+func container(svc string) string {
+	project := "clustertests"
+	if p := os.Getenv("PROJECT"); p != "" {
+		project = p
+	}
+	return project + "_" + svc + "_1"
+}
+
 func TestClusterStuff(t *testing.T) {
 	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
 		t.Skip("pilosa cluster tests are not enabled")
@@ -70,8 +84,7 @@ func TestClusterStuff(t *testing.T) {
 		}
 	}
 	t.Run("long pause", func(t *testing.T) {
-
-		pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s")
+		pcmd := exec.Command("/pumba", "pause", container("pilosa3"), "--duration", "10s")
 		pcmd.Stdout = os.Stdout
 		pcmd.Stderr = os.Stderr
 		t.Log("pausing pilosa3 for 10s")
@@ -101,7 +114,7 @@ func TestClusterStuff(t *testing.T) {
 
 	t.Run("backup", func(t *testing.T) {
 		// do backup with node 1 down, but restart it after a few seconds
-		if err := sendCmd("docker", "stop", "clustertests_pilosa1_1"); err != nil {
+		if err := sendCmd("docker", "stop", container("pilosa1")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
 		var backupCmd *exec.Cmd
@@ -111,7 +124,7 @@ func TestClusterStuff(t *testing.T) {
 			t.Fatalf("sending backup command: %v", err)
 		}
 		time.Sleep(time.Second * 5)
-		if err = sendCmd("docker", "start", "clustertests_pilosa1_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa1")); err != nil {
 			t.Fatalf("sending start command: %v", err)
 		}
 
@@ -137,12 +150,12 @@ func TestClusterStuff(t *testing.T) {
 			t.Fatalf("starting restore: %v", err)
 		}
 		time.Sleep(time.Millisecond * 50)
-		if err = sendCmd("docker", "stop", "clustertests_pilosa2_1"); err != nil {
+		if err = sendCmd("docker", "stop", container("pilosa2")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
 
 		time.Sleep(time.Second * 10)
-		if err = sendCmd("docker", "start", "clustertests_pilosa2_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa2")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
 		if err := restoreCmd.Wait(); err != nil {
@@ -157,25 +170,25 @@ func TestClusterStuff(t *testing.T) {
 			t.Fatalf("sending second backup command: %v", err)
 		}
 		time.Sleep(time.Millisecond * 10) // want the backup to get started, then fail
-		if err = sendCmd("docker", "stop", "clustertests_pilosa1_1"); err != nil {
+		if err = sendCmd("docker", "stop", container("pilosa1")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
-		if err = sendCmd("docker", "stop", "clustertests_pilosa2_1"); err != nil {
+		if err = sendCmd("docker", "stop", container("pilosa2")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
-		if err = sendCmd("docker", "stop", "clustertests_pilosa3_1"); err != nil {
+		if err = sendCmd("docker", "stop", container("pilosa3")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
 
 		time.Sleep(time.Second * 5)
 
-		if err = sendCmd("docker", "start", "clustertests_pilosa1_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa1")); err != nil {
 			t.Fatalf("sending start command: %v", err)
 		}
-		if err = sendCmd("docker", "start", "clustertests_pilosa2_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa2")); err != nil {
 			t.Fatalf("sending start command: %v", err)
 		}
-		if err = sendCmd("docker", "start", "clustertests_pilosa3_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa3")); err != nil {
 			t.Fatalf("sending start command: %v", err)
 		}
 		if err = backupCmd.Wait(); err == nil {

internal/clustertests/docker-compose.yml

@@ -64,6 +64,7 @@ services:
     environment:
       - ENABLE_PILOSA_CLUSTER_TESTS=1
       - GO111MODULE=on
+      - PROJECT=${PROJECT}
     networks:
       - pilosanet
     volumes:

internal/clustertests/pause_node_test.go

@@ -44,12 +44,12 @@ func sendCmd(cmd string, args ...string) error {
 }
 
 func unpauseNode(node string) error {
-	unpauseArgs := []string{"container", "unpause", "clustertests_" + node + "_1"}
+	unpauseArgs := []string{"container", "unpause", container(node)}
 	return sendCmd("docker", unpauseArgs...)
 }
 
 func pauseNode(node string) error {
-	pauseArgs := []string{"container", "pause", "clustertests_" + node + "_1"}
+	pauseArgs := []string{"container", "pause", container(node)}
 	return sendCmd("docker", pauseArgs...)
 }