From e297a6775de9a0f7041de832f06cc867ea099015 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 17 Jan 2022 09:39:28 -0600 Subject: [PATCH 1/4] have simulacradata tests clean up generated files --- .gitignore | 1 + qa/simulacraData/simulacra_data_test.go | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6758e781b..675ffe9df 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ launch.json __pycache__/ report.xml outputs.json +builds/ \ No newline at end of file diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go index ae16de547..c5d7e59ae 100644 --- a/qa/simulacraData/simulacra_data_test.go +++ b/qa/simulacraData/simulacra_data_test.go @@ -2,12 +2,14 @@ package main import ( + "os" "testing" ) const testRecords int = 1000 func TestAge(t *testing.T) { + defer os.Remove("age.csv") err := GenerateAgeField(testRecords) if err != nil { t.Fatalf("%v", err) @@ -15,13 +17,15 @@ func TestAge(t *testing.T) { } func TestIP(t *testing.T) { + defer os.Remove("ip.csv") err := GenerateIPField(testRecords) if err != nil { t.Fatalf("%v", err) } } -func TestIndentifer(t *testing.T) { +func TestIdentifier(t *testing.T) { + defer os.Remove("identifier.csv") err := GenerateArbIdField(testRecords) if err != nil { t.Fatalf("%v", err) @@ -29,6 +33,7 @@ func TestIndentifer(t *testing.T) { } func TestOptIn(t *testing.T) { + defer os.Remove("optin.csv") err := GenerateOptInField(testRecords) if err != nil { t.Fatalf("%v", err) @@ -36,6 +41,7 @@ func TestOptIn(t *testing.T) { } func TestCountry(t *testing.T) { + defer os.Remove("country.csv") err := GenerateCountryField(testRecords) if err != nil { t.Fatalf("%v", err) @@ -43,6 +49,7 @@ func TestCountry(t *testing.T) { } func TestTime(t *testing.T) { + defer os.Remove("time.csv") err := GenerateTimeField(testRecords) if err != nil { t.Fatalf("%v", err) From da03e3fad256832009a94ca3e0a98bf7e3e4770e Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 17 Jan 2022 10:12:40 -0600 Subject: [PATCH 2/4] add race and shardwidth22 to Gitlab CI, cleanup our coverage reporting was a bit wonky and had files coming from both test and test-future... made everything come from future --- .gitlab/.gitlab-ci.yml | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 4164ba8da..6dbd26dc8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,7 +12,7 @@ include: - .go/pkg/mod/ variables: - GOVERSION: "1.16.10" + GOVERSION: "1.16.13" stages: - lint @@ -76,15 +76,34 @@ run go tests: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase unit tests..." - - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) - - go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... - artifacts: - paths: - - coverage.out + - go test ./... +run go tests race: + stage: test + image: golang:$GOVERSION + extends: .go-cache + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - echo "Running featurebase race tests..." + - go test -race -timeout=30m ./... + +run go tests shardwidth22: + stage: test + image: golang:$GOVERSION + extends: .go-cache + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + script: + - echo "Running featurebase race tests..." + - go test -tags=shardwidth22 ./... + +# we do coverage reporting from the future tests because the json +# output is very difficult to human-read. The alternative would be to +# run the regular tests twice and also run the future tests. run go tests future: stage: test - image: golang:1.17.3 + image: golang:1.17.6 extends: .go-cache rules: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' @@ -107,7 +126,7 @@ upload to sonarcloud: script: - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info needs: - - job: run go tests + - job: run go tests future - job: run jest tests build for linux amd64: @@ -211,6 +230,7 @@ build container fb: # 3. make sure docker/docker-compose is installed # 4. make sure the git config is done `git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"` # 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user +# TODO: (I think) get clustertests coverage added to coverage report clustertests: stage: integration tags: @@ -224,6 +244,7 @@ clustertests: external lookup tests: stage: integration image: golang:$GOVERSION + # TODO: no rules here, do we need to add the rules line? variables: POSTGRES_DB: $POSTGRES_DB POSTGRES_USER: $POSTGRES_USER From a16fee5f88efc2ca85e93b4c8d257b8b2ed2cd05 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 17 Jan 2022 10:57:01 -0600 Subject: [PATCH 3/4] set shardWidth properly in client the shardwidth22 tests were broken client side, but we didn't realize this because we weren't running the client side tests since moving the client code into the main FB repo until recently (woops), and more recently, we'd stopped running the shardwidth22 tests in the move to Gitlab, so when we re-enabled them we finally noticed that they were broken in the client. All this change does is takes the shardWidth value from the core featurebase package instead of using a hardcoded value in the client package. --- client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/client.go b/client/client.go index 211e7c3f7..082a56634 100644 --- a/client/client.go +++ b/client/client.go @@ -36,7 +36,7 @@ import ( const PQLVersion = "1.0" // DefaultShardWidth is used if an index doesn't have it defined. -const DefaultShardWidth = 1 << 20 +const DefaultShardWidth = pilosa.ShardWidth const maxHosts = 10 From fdf7b4107a16fe45e60639ad4d372219b372c0ac Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 18 Jan 2022 12:14:54 -0600 Subject: [PATCH 4/4] actually be able to generate-statik these were the changes i had to make to be able to build lattice on my machine --- lattice/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lattice/Dockerfile b/lattice/Dockerfile index 2b30d6cfb..6aa669e54 100644 --- a/lattice/Dockerfile +++ b/lattice/Dockerfile @@ -1,10 +1,10 @@ FROM moleculacorp/nodejs:latest as build - +# make sure that your docker settings allow for at least like 4gb of ram, it +# takes a lot to build this WORKDIR /lattice - COPY package.json ./ -COPY yarn.lock ./ -RUN yarn install +RUN apk update && apk upgrade yarn +RUN yarn install --network-timeout 100000 COPY . ./ RUN yarn build