Merge pull request #1913 from molecula/clustertests-coverage

FB-1183: Enable code coverage for clustertests

.gitlab/.gitlab-ci.yml

@@ -126,17 +126,18 @@ run go tests future:
     - aws
 
 upload to sonarcloud:
-  stage: test
+  stage: integration
   image: sonarsource/sonar-scanner-cli:4.6
   variables:
     SONAR_TOKEN: $SONAR_TOKEN
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:
-    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
+    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report*  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
   needs:
     - job: run go tests future
     - job: run jest tests 
+    - job: clustertests
     
 build for linux amd64:
   stage: build
@@ -289,7 +290,12 @@ clustertests:
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:
+    - rm -rf  internal/clustertests/results && mkdir -p  internal/clustertests/results && chown gitlab-runner:gitlab-runner  internal/clustertests/results
     - make clustertests
+    - mv internal/clustertests/results/ results/
+  artifacts:
+    paths:
+      - results/coverage*.out
 
 authclustertests:
   variables:
@@ -302,6 +308,7 @@ authclustertests:
   script:
     - make authclustertests
 
+
 external lookup tests:
   stage: integration
   image: golang:$GOVERSION

Dockerfile-clustertests

@@ -7,8 +7,6 @@ LABEL maintainer "dev@pilosa.com"
 
 COPY . /go/src/github.com/molecula/featurebase/
 
-RUN cd /go/src/github.com/molecula/featurebase \
-    && make install FLAGS="-a -mod=vendor"
 
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
@@ -22,7 +20,11 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
 
-RUN cp /go/bin/featurebase /featurebase
+# generate an instrumented binary to allow for calculating code coverage for clustertests
+# the entrypoint for the binary is TestRunMain, which is wrapper for main 
+RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
+    go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
+    cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
 
 COPY NOTICE /NOTICE
 
@@ -30,4 +32,4 @@ EXPOSE 10101
 VOLUME /data
 
 ENTRYPOINT ["bash", "-c"]
-CMD ["/featurebase", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

Dockerfile-clustertests-client

@@ -7,9 +7,6 @@ LABEL maintainer "dev@pilosa.com"
 
 COPY . /go/src/github.com/molecula/featurebase/
 
-RUN cd /go/src/github.com/molecula/featurebase \
-    && make install FLAGS="-a -mod=vendor"
-
 # download pumba for fault injection
 ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
 RUN chmod +x /pumba
@@ -22,7 +19,10 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose
 
-RUN cp /go/bin/featurebase /featurebase
+RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
+    go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
+    cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
+
 
 COPY NOTICE /NOTICE
 
@@ -32,4 +32,4 @@ EXPOSE 10101
 VOLUME /data
 
 ENTRYPOINT ["bash", "-c"]
-CMD ["/featurebase", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
+CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

cmd/featurebase/main_test.go

@@ -0,0 +1,13 @@
+//go:build testrunmain
+// +build testrunmain
+
+package main
+
+import (
+	"testing"
+)
+
+// Wrapper test for main function used to get code coverage for end2end tests
+func TestRunMain(t *testing.T) {
+	main()
+}

internal/clustertests/cluster_test.go

@@ -2,6 +2,7 @@
 package clustertest
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"fmt"
@@ -16,6 +17,7 @@ import (
 	"github.com/golang-jwt/jwt"
 	pilosa "github.com/molecula/featurebase/v3"
 	"github.com/molecula/featurebase/v3/authn"
+	"github.com/molecula/featurebase/v3/ctl"
 	"github.com/molecula/featurebase/v3/disco"
 	"github.com/molecula/featurebase/v3/encoding/proto"
 	"github.com/molecula/featurebase/v3/logger"
@@ -178,17 +180,18 @@ func TestClusterStuff(t *testing.T) {
 		var backupCmd *exec.Cmd
 		tmpdir := t.TempDir()
 
+		// collect code coverage while doing backup using an instrumented binary by calling
+		// a wrapper test (TestRunMain) for the main entrypoint of featurebase
+		args := []string{"-test.run=TestRunMain", "-test.coverprofile=/results/coverage-backup.out", "backup",
+			"--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest")}
 		if auth {
-			if backupCmd, err = startCmd(
-				"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest"), "--auth-token", token); err != nil {
-				t.Fatalf("sending backup command: %v", err)
-			}
-		} else {
-			if backupCmd, err = startCmd(
-				"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest")); err != nil {
-				t.Fatalf("sending backup command: %v", err)
-			}
+			args = append(args, fmt.Sprintf("--auth-token=%s", token))
 		}
+
+		if backupCmd, err = startCmd("/featurebase", args...); err != nil {
+			t.Fatalf("sending backup command: %v", err)
+		}
+
 		time.Sleep(time.Second * 5)
 		if err = sendCmd("docker", "start", container(t, "pilosa1")); err != nil {
 			t.Fatalf("sending start command: %v", err)
@@ -216,15 +219,15 @@ func TestClusterStuff(t *testing.T) {
 		}
 
 		var restoreCmd *exec.Cmd
+		args = []string{"-test.run=TestRunMain", "-test.coverprofile=/results/coverage-restore.out", "restore",
+			"-s", tmpdir + "/backuptest", "--host", "pilosa1:10101"}
 		if auth {
-			if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101", "--auth-token", token); err != nil {
-				t.Fatalf("starting restore: %v", err)
-			}
-		} else {
-			if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101"); err != nil {
-				t.Fatalf("starting restore: %v", err)
-			}
+			args = append(args, fmt.Sprintf("--auth-token=%s", token))
 		}
+		if restoreCmd, err = startCmd("/featurebase", args...); err != nil {
+			t.Fatalf("starting restore: %v", err)
+		}
+
 		time.Sleep(time.Millisecond * 50)
 		if err = sendCmd("docker", "stop", container(t, "pilosa2")); err != nil {
 			t.Fatalf("sending stop command: %v", err)
@@ -250,16 +253,23 @@ func TestClusterStuff(t *testing.T) {
 		// now do backup with all nodes down and too short a timeout
 		// so it fails. Has be to be all 3 because the cluster has
 		// replicas=3 and the backup command will retry on replicas.
+		// featurebase backup cmd can't be used for a test expected to fail
+		// because code coverage report won't be generated.
+		buf := bytes.Buffer{}
+		rder := []byte{}
+		stdin := bytes.NewReader(rder)
+		stdout := bufio.NewWriter(&buf)
+		stderr := bufio.NewWriter(&buf)
+		backup := ctl.NewBackupCommand(stdin, stdout, stderr)
+		backup.Host = "--host=pilosa1:10101"
+		backup.OutputDir = tmpdir + "/backuptest2"
+		backup.RetryPeriod = time.Millisecond * 200
 		if auth {
-			if backupCmd, err = startCmd(
-				"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms", "--auth-token", token); err != nil {
-				t.Fatalf("sending second backup command: %v", err)
-			}
-		} else {
-			if backupCmd, err = startCmd(
-				"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms"); err != nil {
-				t.Fatalf("sending second backup command: %v", err)
-			}
+			backup.AuthToken = token
+		}
+
+		if err = backup.Run(context.Background()); err == nil {
+			t.Fatal("backup command should have errored but didn't")
 		}
 
 		t.Logf("sleeping 8s")
@@ -275,10 +285,6 @@ func TestClusterStuff(t *testing.T) {
 		if err = sendCmd("docker", "unpause", container(t, "pilosa3")); err != nil {
 			t.Fatalf("sending unpause command: %v", err)
 		}
-		if err = backupCmd.Wait(); err == nil {
-			t.Fatal("backup command should have errored but didn't")
-		}
-
 	})
 }
 

internal/clustertests/docker-compose.yml

@@ -15,8 +15,10 @@ services:
       - PILOSA_CLUSTER_REPLICAS=3
     networks:
       - pilosanet
+    volumes:
+      - ./results:/results 
     command:
-      - "/featurebase server --bind pilosa1:10101 ${CLUSTERTESTS_FB_ARGS}"
+      - "cd /go/src/github.com/molecula/featurebase/cmd/featurebase && /featurebase -test.run=TestRunMain -test.coverprofile=/results/coverage-server1.out server --bind pilosa1:10101 ${CLUSTERTESTS_FB_ARGS}"
   pilosa2:
     build:
       context: ../..
@@ -32,8 +34,10 @@ services:
       - PILOSA_CLUSTER_REPLICAS=3
     networks:
       - pilosanet
+    volumes:
+      - ./results:/results
     command:
-      - "/featurebase server --bind pilosa2:10101 ${CLUSTERTESTS_FB_ARGS}"
+      - "cd /go/src/github.com/molecula/featurebase/cmd/featurebase && /featurebase -test.run=TestRunMain -test.coverprofile=/results/coverage-server2.out server --bind pilosa2:10101 ${CLUSTERTESTS_FB_ARGS}"
   pilosa3:
     build:
       context: ../..
@@ -49,8 +53,10 @@ services:
       - PILOSA_CLUSTER_REPLICAS=3
     networks:
       - pilosanet
+    volumes:
+      - ./results:/results
     command:
-      - "/featurebase server --bind pilosa3:10101 ${CLUSTERTESTS_FB_ARGS}"
+      - "cd /go/src/github.com/molecula/featurebase/cmd/featurebase && /featurebase -test.run=TestRunMain -test.coverprofile=/results/coverage-server3.out server --bind pilosa3:10101 ${CLUSTERTESTS_FB_ARGS}"
   client1:
     build:
       context: ../..
@@ -69,8 +75,9 @@ services:
       - pilosanet
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      - ./results:/results
     command:
-      - "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -count=1 github.com/molecula/featurebase/v3/internal/clustertests"
+      - "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -count=1 -covermode=atomic -coverprofile=/results/coverage-clustertests.out -coverpkg=./... -json github.com/molecula/featurebase/v3/internal/clustertests | tee /results/report-clustertests.out"
   fakeidp:
     build:
       context: .