Merge branch 'master' into cicd-will-it-never-end

.gitlab/.gitlab-ci.yml

@@ -259,10 +259,11 @@ build arm container fb:
 # 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user
 # TODO: (I think) get clustertests coverage added to coverage report
 clustertests:
+  variables:
+    PROJECT: clustertests_${CI_CONCURRENT_ID}
   stage: integration
   tags:
     - shell
-    - gcp # this is to restrict to the GCP runner we set up manually, once all our runners are set up properly we can remove this.
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:

Dockerfile-clustertests

@@ -18,6 +18,10 @@ RUN chmod +x /pumba
 RUN apt update
 RUN apt install -y docker.io 
 
+# add docker-compose so tests can use it for stuff
+ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
+RUN chmod +x /usr/local/bin/docker-compose
+
 RUN cp /go/bin/featurebase /featurebase
 
 COPY NOTICE /NOTICE

Makefile

@@ -140,19 +140,23 @@ package:
 	nfpm package --packager deb --target featurebase_$(VERSION_ID).deb
 	nfpm package --packager rpm --target featurebase_$(VERSION_ID).rpm
 
-# try (e.g.) internal/clustertests/docker-compose-replication2.yml
-DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
+
+# We allow setting a custom docker-compose "project". Multiple of the
+# same docker-compose environment can exist simultaneously as long as
+# they use different projects (the project name is prepended to
+# container names and such). This is useful in a CI environment where
+# we might be running multiple instances of the tests concurrently.
+PROJECT ?= clustertests
+DOCKER_COMPOSE = docker-compose -p $(PROJECT)
 
 # Run cluster integration tests using docker. Requires docker daemon to be
-# running. This will catch changes to internal/clustertests/*.go, but if you
-# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
-# pilosa image.
+# running and docker-compose to be installed.
 clustertests: vendor
-	docker-compose -f $(DOCKER_COMPOSE) down
-	docker-compose -f $(DOCKER_COMPOSE) build
-	docker-compose -f $(DOCKER_COMPOSE) up -d pilosa1 pilosa2 pilosa3
-	docker-compose -f $(DOCKER_COMPOSE) run client1
-	docker-compose -f $(DOCKER_COMPOSE) down
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
+	PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
+	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
 
 
 # Install Pilosa

api.go

@@ -2514,24 +2514,24 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du
 		return nil, errors.Wrap(err, "validating api method")
 	}
 	t, err := api.server.StartTransaction(ctx, id, timeout, exclusive, remote)
-	if exclusive {
-		switch err {
-		case nil:
+
+	switch err {
+	case nil:
+		if exclusive {
 			api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0)
-		case ErrTransactionExclusive:
-			api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
-		}
-		if t.Active {
-			api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
-		}
-	} else {
-		switch err {
-		case nil:
+		} else {
 			api.holder.Stats.Count(MetricTransactionStart, 1, 1.0)
-		case ErrTransactionExclusive:
+		}
+	case ErrTransactionExclusive:
+		if exclusive {
+			api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
+		} else {
 			api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0)
 		}
 	}
+	if exclusive && t != nil && t.Active {
+		api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
+	}
 	return t, err
 }
 

http/client_test.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
-	"github.com/molecula/featurebase/v3"
+	pilosa "github.com/molecula/featurebase/v3"
 	"github.com/molecula/featurebase/v3/http"
 	"github.com/molecula/featurebase/v3/pql"
 	"github.com/molecula/featurebase/v3/server"
@@ -1419,12 +1419,11 @@ func TestClientTransactions(t *testing.T) {
 	}
 
 	// non-primary
-	if trns, err := client1.StartTransaction(context.Background(), "blah", time.Minute, false); err == nil ||
-		!strings.Contains(err.Error(), pilosa.ErrNodeNotPrimary.Error()) {
+	if trns, err := client1.StartTransaction(context.Background(), "blah", time.Minute, false); err != nil {
 		t.Fatalf("unexpected error starting on non-primary: %v", err)
 	} else {
 		test.CompareTransactions(t,
-			nil,
+			&pilosa.Transaction{ID: "blah", Timeout: time.Minute, Active: true, Exclusive: false, Deadline: expDeadline},
 			trns)
 	}
 

http/handler.go

@@ -2158,9 +2158,15 @@ func (h *Handler) handlePostTransaction(w http.ResponseWriter, r *http.Request)
 	if !ok {
 		id = reqTrns.ID
 	}
-	trns, err := h.api.StartTransaction(r.Context(), id, reqTrns.Timeout, reqTrns.Exclusive, false)
 
-	h.doTransactionResponse(w, err, trns)
+	if primary := h.api.PrimaryNode(); h.api.NodeID() == primary.ID {
+		trns, err := h.api.StartTransaction(r.Context(), id, reqTrns.Timeout, reqTrns.Exclusive, false)
+		h.doTransactionResponse(w, err, trns)
+		return
+	} else {
+		http.Redirect(w, r, primary.URI.Normalize()+"/transaction/"+id, http.StatusSeeOther)
+		return
+	}
 }
 
 func (h *Handler) handlePostFinishTransaction(w http.ResponseWriter, r *http.Request) {

internal/clustertests/cluster_test.go

@@ -16,6 +16,20 @@ import (
 	picli "github.com/molecula/featurebase/v3/http"
 )
 
+// container turns a docker-compose service name into a container name
+// assuming the project name is set in the enviroment as PROJECT. This
+// refers to the "-p" argument to docker-compose. NOTE: this assumes
+// docker-compose joins the project name with a separating
+// underscore... this may not always be true as I've seen a dash used
+// as well, but I think it is true in recent versions.
+func container(svc string) string {
+	project := "clustertests"
+	if p := os.Getenv("PROJECT"); p != "" {
+		project = p
+	}
+	return project + "_" + svc + "_1"
+}
+
 func TestClusterStuff(t *testing.T) {
 	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
 		t.Skip("pilosa cluster tests are not enabled")
@@ -70,8 +84,7 @@ func TestClusterStuff(t *testing.T) {
 		}
 	}
 	t.Run("long pause", func(t *testing.T) {
-
-		pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s")
+		pcmd := exec.Command("/pumba", "pause", container("pilosa3"), "--duration", "10s")
 		pcmd.Stdout = os.Stdout
 		pcmd.Stderr = os.Stderr
 		t.Log("pausing pilosa3 for 10s")
@@ -101,7 +114,7 @@ func TestClusterStuff(t *testing.T) {
 
 	t.Run("backup", func(t *testing.T) {
 		// do backup with node 1 down, but restart it after a few seconds
-		if err := sendCmd("docker", "stop", "clustertests_pilosa1_1"); err != nil {
+		if err := sendCmd("docker", "stop", container("pilosa1")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
 		var backupCmd *exec.Cmd
@@ -111,7 +124,7 @@ func TestClusterStuff(t *testing.T) {
 			t.Fatalf("sending backup command: %v", err)
 		}
 		time.Sleep(time.Second * 5)
-		if err = sendCmd("docker", "start", "clustertests_pilosa1_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa1")); err != nil {
 			t.Fatalf("sending start command: %v", err)
 		}
 
@@ -137,12 +150,12 @@ func TestClusterStuff(t *testing.T) {
 			t.Fatalf("starting restore: %v", err)
 		}
 		time.Sleep(time.Millisecond * 50)
-		if err = sendCmd("docker", "stop", "clustertests_pilosa2_1"); err != nil {
+		if err = sendCmd("docker", "stop", container("pilosa2")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
 
 		time.Sleep(time.Second * 10)
-		if err = sendCmd("docker", "start", "clustertests_pilosa2_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa2")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
 		if err := restoreCmd.Wait(); err != nil {
@@ -157,25 +170,25 @@ func TestClusterStuff(t *testing.T) {
 			t.Fatalf("sending second backup command: %v", err)
 		}
 		time.Sleep(time.Millisecond * 10) // want the backup to get started, then fail
-		if err = sendCmd("docker", "stop", "clustertests_pilosa1_1"); err != nil {
+		if err = sendCmd("docker", "stop", container("pilosa1")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
-		if err = sendCmd("docker", "stop", "clustertests_pilosa2_1"); err != nil {
+		if err = sendCmd("docker", "stop", container("pilosa2")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
-		if err = sendCmd("docker", "stop", "clustertests_pilosa3_1"); err != nil {
+		if err = sendCmd("docker", "stop", container("pilosa3")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
 		}
 
 		time.Sleep(time.Second * 5)
 
-		if err = sendCmd("docker", "start", "clustertests_pilosa1_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa1")); err != nil {
 			t.Fatalf("sending start command: %v", err)
 		}
-		if err = sendCmd("docker", "start", "clustertests_pilosa2_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa2")); err != nil {
 			t.Fatalf("sending start command: %v", err)
 		}
-		if err = sendCmd("docker", "start", "clustertests_pilosa3_1"); err != nil {
+		if err = sendCmd("docker", "start", container("pilosa3")); err != nil {
 			t.Fatalf("sending start command: %v", err)
 		}
 		if err = backupCmd.Wait(); err == nil {

internal/clustertests/docker-compose-replication2.yml

@@ -5,8 +5,6 @@ services:
       context: ../..
       dockerfile: Dockerfile-clustertests
     image: ptest
-    ports:
-      - "33455:10101"
     environment:
       - PILOSA_CLUSTER_COORDINATOR=true
       - PILOSA_GOSSIP_SEEDS=pilosa1:14000
@@ -20,8 +18,6 @@ services:
       context: ../..
       dockerfile: Dockerfile-clustertests
     image: ptest
-    ports:
-      - "33456:10101"
     environment:
       - PILOSA_GOSSIP_SEEDS=pilosa1:14000
       - PILOSA_CLUSTER_REPLICAS=2
@@ -34,8 +30,6 @@ services:
       context: ../..
       dockerfile: Dockerfile-clustertests
     image: ptest
-    ports:
-      - "33457:10101"
     environment:
       - PILOSA_GOSSIP_SEEDS=pilosa1:14000,pilosa2:14000
       - PILOSA_CLUSTER_REPLICAS=2

internal/clustertests/docker-compose.yml

@@ -5,8 +5,6 @@ services:
       context: ../..
       dockerfile: Dockerfile-clustertests
     image: ptest
-    ports:
-      - "33455:10101"
     environment:
       - PILOSA_NAME=pilosa1
       - PILOSA_ETCD_DIR=/root/.etcd
@@ -25,8 +23,6 @@ services:
       context: ../..
       dockerfile: Dockerfile-clustertests
     image: ptest
-    ports:
-      - "33456:10101"
     environment:
       - PILOSA_NAME=pilosa2
       - PILOSA_ETCD_DIR=/root/.etcd
@@ -45,8 +41,6 @@ services:
       context: ../..
       dockerfile: Dockerfile-clustertests
     image: ptest
-    ports:
-      - "33457:10101"
     environment:
       - PILOSA_NAME=pilosa3
       - PILOSA_ETCD_DIR=/root/.etcd
@@ -70,11 +64,13 @@ services:
     environment:
       - ENABLE_PILOSA_CLUSTER_TESTS=1
       - GO111MODULE=on
+      - PROJECT=${PROJECT}
     networks:
       - pilosanet
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     command:
       - "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -count=1 github.com/molecula/featurebase/v3/internal/clustertests"
+
 networks: 
   pilosanet:

internal/clustertests/pause_node_test.go

@@ -44,12 +44,12 @@ func sendCmd(cmd string, args ...string) error {
 }
 
 func unpauseNode(node string) error {
-	unpauseArgs := []string{"container", "unpause", "clustertests_" + node + "_1"}
+	unpauseArgs := []string{"container", "unpause", container(node)}
 	return sendCmd("docker", unpauseArgs...)
 }
 
 func pauseNode(node string) error {
-	pauseArgs := []string{"container", "pause", "clustertests_" + node + "_1"}
+	pauseArgs := []string{"container", "pause", container(node)}
 	return sendCmd("docker", pauseArgs...)
 }