Merge branch 'master' into cicd-will-it-never-end

.gitignore

@@ -20,3 +20,4 @@ launch.json
 __pycache__/
 report.xml
 outputs.json
+builds/

.gitlab/.gitlab-ci.yml

@@ -12,7 +12,7 @@ include:
           - .go/pkg/mod/
 
 variables:
-  GOVERSION: "1.16.10"
+  GOVERSION: "1.16.13"
 
 stages:
   - lint
@@ -76,15 +76,34 @@ run go tests:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
   script:
     - echo "Running featurebase unit tests..."
-    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
-    - go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./...
-  artifacts:
-    paths:
-      - coverage.out
+    - go test ./...
 
+run go tests race:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Running featurebase race tests..."
+    - go test -race -timeout=30m ./...
+
+run go tests shardwidth22:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Running featurebase race tests..."
+    - go test -tags=shardwidth22 ./...
+    
+# we do coverage reporting from the future tests because the json
+# output is very difficult to human-read. The alternative would be to
+# run the regular tests twice and also run the future tests.
 run go tests future:
   stage: test
-  image: golang:1.17.3
+  image: golang:1.17.6
   extends: .go-cache
   rules:
     - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
@@ -107,7 +126,7 @@ upload to sonarcloud:
   script:
     - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
   needs:
-    - job: run go tests
+    - job: run go tests future
     - job: run jest tests 
     
 build for linux amd64:
@@ -243,6 +262,7 @@ build arm container fb:
 # 3. make sure docker/docker-compose is installed
 # 4. make sure the git config is done `git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"`
 # 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user
+# TODO: (I think) get clustertests coverage added to coverage report
 clustertests:
   stage: integration
   tags:
@@ -256,6 +276,7 @@ clustertests:
 external lookup tests:
   stage: integration
   image: golang:$GOVERSION
+  # TODO: no rules here, do we need to add the rules line?
   variables:
     POSTGRES_DB: $POSTGRES_DB
     POSTGRES_USER: $POSTGRES_USER

client/client.go

@@ -36,7 +36,7 @@ import (
 const PQLVersion = "1.0"
 
 // DefaultShardWidth is used if an index doesn't have it defined.
-const DefaultShardWidth = 1 << 20
+const DefaultShardWidth = pilosa.ShardWidth
 
 const maxHosts = 10
 

lattice/Dockerfile

@@ -1,10 +1,10 @@
 FROM moleculacorp/nodejs:latest as build
-
+# make sure that your docker settings allow for at least like 4gb of ram, it
+# takes a lot to build this
 WORKDIR /lattice
-
 COPY package.json ./
-COPY yarn.lock ./
-RUN yarn install
+RUN apk update && apk upgrade yarn
+RUN yarn install --network-timeout 100000
 
 COPY . ./
 RUN yarn build

qa/simulacraData/simulacra_data_test.go

@@ -2,12 +2,14 @@
 package main
 
 import (
+	"os"
 	"testing"
 )
 
 const testRecords int = 1000
 
 func TestAge(t *testing.T) {
+	defer os.Remove("age.csv")
 	err := GenerateAgeField(testRecords)
 	if err != nil {
 		t.Fatalf("%v", err)
@@ -15,13 +17,15 @@ func TestAge(t *testing.T) {
 }
 
 func TestIP(t *testing.T) {
+	defer os.Remove("ip.csv")
 	err := GenerateIPField(testRecords)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
 }
 
-func TestIndentifer(t *testing.T) {
+func TestIdentifier(t *testing.T) {
+	defer os.Remove("identifier.csv")
 	err := GenerateArbIdField(testRecords)
 	if err != nil {
 		t.Fatalf("%v", err)
@@ -29,6 +33,7 @@ func TestIndentifer(t *testing.T) {
 }
 
 func TestOptIn(t *testing.T) {
+	defer os.Remove("optin.csv")
 	err := GenerateOptInField(testRecords)
 	if err != nil {
 		t.Fatalf("%v", err)
@@ -36,6 +41,7 @@ func TestOptIn(t *testing.T) {
 }
 
 func TestCountry(t *testing.T) {
+	defer os.Remove("country.csv")
 	err := GenerateCountryField(testRecords)
 	if err != nil {
 		t.Fatalf("%v", err)
@@ -43,6 +49,7 @@ func TestCountry(t *testing.T) {
 }
 
 func TestTime(t *testing.T) {
+	defer os.Remove("time.csv")
 	err := GenerateTimeField(testRecords)
 	if err != nil {
 		t.Fatalf("%v", err)