diff --git a/.gitignore b/.gitignore index 6758e781b..675ffe9df 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ launch.json __pycache__/ report.xml outputs.json +builds/ \ No newline at end of file diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 92023b865..7a3a56e49 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,7 +12,7 @@ include: - .go/pkg/mod/ variables: - GOVERSION: "1.16.10" + GOVERSION: "1.16.13" stages: - lint @@ -76,15 +76,34 @@ run go tests: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase unit tests..." - - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) - - go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... - artifacts: - paths: - - coverage.out + - go test ./... +run go tests race: + stage: test + image: golang:$GOVERSION + extends: .go-cache + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - echo "Running featurebase race tests..." + - go test -race -timeout=30m ./... + +run go tests shardwidth22: + stage: test + image: golang:$GOVERSION + extends: .go-cache + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - echo "Running featurebase race tests..." + - go test -tags=shardwidth22 ./... + +# we do coverage reporting from the future tests because the json +# output is very difficult to human-read. The alternative would be to +# run the regular tests twice and also run the future tests. run go tests future: stage: test - image: golang:1.17.3 + image: golang:1.17.6 extends: .go-cache rules: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' @@ -107,7 +126,7 @@ upload to sonarcloud: script: - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info needs: - - job: run go tests + - job: run go tests future - job: run jest tests build for linux amd64: @@ -243,6 +262,7 @@ build arm container fb: # 3. make sure docker/docker-compose is installed # 4. make sure the git config is done `git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"` # 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user +# TODO: (I think) get clustertests coverage added to coverage report clustertests: stage: integration tags: @@ -256,6 +276,7 @@ clustertests: external lookup tests: stage: integration image: golang:$GOVERSION + # TODO: no rules here, do we need to add the rules line? variables: POSTGRES_DB: $POSTGRES_DB POSTGRES_USER: $POSTGRES_USER diff --git a/client/client.go b/client/client.go index 211e7c3f7..082a56634 100644 --- a/client/client.go +++ b/client/client.go @@ -36,7 +36,7 @@ import ( const PQLVersion = "1.0" // DefaultShardWidth is used if an index doesn't have it defined. -const DefaultShardWidth = 1 << 20 +const DefaultShardWidth = pilosa.ShardWidth const maxHosts = 10 diff --git a/lattice/Dockerfile b/lattice/Dockerfile index 2b30d6cfb..6aa669e54 100644 --- a/lattice/Dockerfile +++ b/lattice/Dockerfile @@ -1,10 +1,10 @@ FROM moleculacorp/nodejs:latest as build - +# make sure that your docker settings allow for at least like 4gb of ram, it +# takes a lot to build this WORKDIR /lattice - COPY package.json ./ -COPY yarn.lock ./ -RUN yarn install +RUN apk update && apk upgrade yarn +RUN yarn install --network-timeout 100000 COPY . ./ RUN yarn build diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go index ae16de547..c5d7e59ae 100644 --- a/qa/simulacraData/simulacra_data_test.go +++ b/qa/simulacraData/simulacra_data_test.go @@ -2,12 +2,14 @@ package main import ( + "os" "testing" ) const testRecords int = 1000 func TestAge(t *testing.T) { + defer os.Remove("age.csv") err := GenerateAgeField(testRecords) if err != nil { t.Fatalf("%v", err) @@ -15,13 +17,15 @@ func TestAge(t *testing.T) { } func TestIP(t *testing.T) { + defer os.Remove("ip.csv") err := GenerateIPField(testRecords) if err != nil { t.Fatalf("%v", err) } } -func TestIndentifer(t *testing.T) { +func TestIdentifier(t *testing.T) { + defer os.Remove("identifier.csv") err := GenerateArbIdField(testRecords) if err != nil { t.Fatalf("%v", err) @@ -29,6 +33,7 @@ func TestIndentifer(t *testing.T) { } func TestOptIn(t *testing.T) { + defer os.Remove("optin.csv") err := GenerateOptInField(testRecords) if err != nil { t.Fatalf("%v", err) @@ -36,6 +41,7 @@ func TestOptIn(t *testing.T) { } func TestCountry(t *testing.T) { + defer os.Remove("country.csv") err := GenerateCountryField(testRecords) if err != nil { t.Fatalf("%v", err) @@ -43,6 +49,7 @@ func TestCountry(t *testing.T) { } func TestTime(t *testing.T) { + defer os.Remove("time.csv") err := GenerateTimeField(testRecords) if err != nil { t.Fatalf("%v", err)