From ebd0fa85fa65d2a5fd9dda26011e365ca1c85159 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 9 Nov 2022 11:47:29 -0600 Subject: [PATCH] shuffle some CI stages, improve IDK test determination mark IDK tests and batch tests as "nonblocking" so we don't wait on them before doing builds. this risks running builds when the IDK or batch tests could fail. It also unrisks spending an extra five minutes in CI waiting for the last "test" phase thing to run in a long dependency chain, which is potentially more significant. Make IDK tests run for changes to idk, client, or batch directories, because at least one IDK thing uses the batch importer. Also, make that actually work, we think -- verified that the IDK tests got run when there were changes in that directory, but there aren't any now, so we don't expect to see them. Also, move the "IDK changed" rules into their own heading and incorporate them by reference, and simplify the conditional because it seemed to be Acting Up, but also make it check against refs/heads/master, rather than possibly just the parent commit, since that seems to be more consistent. Finally, we combine four of the tests (go tests, go tests future, go tests future plg, go tests shardwidth22) into two tests, both using the "future" compiler, one for plg, one for shardwidth22, so we don't need as many parallel runners and are less likely to end up waiting on them. At this point, the largest delay in CI is the chain through building lattice, which blocks some tasks for a fairly long time because they just have to wait for us to have built a container we can use as the server container in tests that need a server to work against. --- .gitlab/.gitlab-ci.yml | 82 +++++++++++++++--------------------------- .gitlab/batch-ci.yml | 6 +++- 2 files changed, 33 insertions(+), 55 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 5aa10064b..a1eecb8e9 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -4,6 +4,16 @@ include: - template: Security/License-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml +.idk_changed: + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + changes: + compare_to: refs/heads/master + paths: + - idk/** + - client/** + - batch/** + gosec-sast: allow_failure: false before_script: @@ -189,19 +199,6 @@ run jest tests: paths: - lattice/coverage/lcov.info -run go tests: - stage: test - image: golang:$GOVERSION - extends: .go-cache - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - retry: 1 - script: - - echo "Running featurebase unit tests..." - - go test -v -timeout=10m $(go list ./... | grep -Ev 'batch|idk') - tags: - - aws - run go tests race: stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests. image: golang:$GOVERSION @@ -216,22 +213,10 @@ run go tests race: tags: - aws -run go tests shardwidth22: - stage: test - image: golang:$GOVERSION - extends: .go-cache - rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' - script: - - echo "Running featurebase shardwidth22 tests..." - - go test -timeout=10m -tags=shardwidth22 -v $(go list ./... | grep -Ev 'batch|idk') - tags: - - aws - -# we do coverage reporting from the future tests because the json -# output is very difficult to human-read. The alternative would be to -# run the regular tests twice and also run the future tests. -run go tests future: +# we dump output in json mode because we're sending the test report data to sonarcloud. it's +# harder to read but easier to use, and running it an extra time without json, when we expect +# tests to succeed, makes CI slow +run go tests: stage: test image: golang:$GOFUTURE extends: .go-cache @@ -241,7 +226,7 @@ run go tests future: script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | grep -Ev 'batch|idk' | paste -s -d, -) - - go test -timeout=10m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev 'batch|idk') | tee test-report.out + - go test -timeout=10m -json -tags=shardwidth22 -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev 'batch|idk') | tee test-report.out artifacts: paths: - coverage.out @@ -249,7 +234,8 @@ run go tests future: tags: - aws -run go tests future plg: +# json mode because test report data goes to sonarcloud +run go tests plg: stage: test image: golang:$GOFUTURE extends: .go-cache @@ -274,13 +260,10 @@ run go tests idk race: variables: USERNAME: fb-idk-access PROJECT: race_${CI_CONCURRENT_ID} - stage: test + stage: nonblocking retry: 1 rules: - - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - changes: - - idk/**/* - - client/**/* + - !reference [.idk_changed, rules] script: - echo "Running test-all-race" - cd ./idk/ @@ -309,13 +292,10 @@ run go tests idk shard transactional: IDK_DEFAULT_SHARD_TRANSACTIONAL: 1 USERNAME: fb-idk-access PROJECT: shardttrans_${CI_CONCURRENT_ID} - stage: test + stage: nonblocking retry: 1 rules: - - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - changes: - - idk/**/* - - client/**/* + - !reference [.idk_changed, rules] script: - echo "Running shard transactional tests" - cd ./idk/ @@ -344,7 +324,7 @@ run go tests idk 533: variables: USERNAME: fb-idk-access PROJECT: test533_${CI_CONCURRENT_ID} - stage: test + stage: nonblocking retry: 1 script: - echo "Running confluent 5.3.3 test-all" @@ -357,10 +337,7 @@ run go tests idk 533: - make save-pilosa-logs - make shutdown rules: - - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - changes: - - idk/**/* - - client/**/* + - !reference [.idk_changed, rules] tags: - shell - aws @@ -377,7 +354,7 @@ run go tests idk sasl: variables: USERNAME: fb-idk-access PROJECT: sasl_${CI_CONCURRENT_ID} - stage: test + stage: nonblocking retry: 1 script: - echo "Running test-all-kafka-sasl" @@ -390,10 +367,7 @@ run go tests idk sasl: - make save-pilosa-logs - make shutdown rules: - - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' - changes: - - idk/**/* - - client/**/* + - !reference [.idk_changed, rules] tags: - shell - aws @@ -406,7 +380,7 @@ run go tests idk sasl: upload to sonarcloud: - stage: integration + stage: nonblocking image: sonarsource/sonar-scanner-cli:4.6 variables: SONAR_TOKEN: $SONAR_TOKEN @@ -415,8 +389,8 @@ upload to sonarcloud: script: - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out,batch/testdata/*coverage.out -Dsonar.go.tests.reportPaths=test-report*.out,idk/testdata/*report.out,batch/testdata/*report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info needs: - - job: run go tests future plg - - job: run go tests future + - job: run go tests + - job: run go tests plg - job: run jest tests - job: external lookup tests - job: run go tests idk race diff --git a/.gitlab/batch-ci.yml b/.gitlab/batch-ci.yml index 7bdd628f8..7b3e47bc3 100644 --- a/.gitlab/batch-ci.yml +++ b/.gitlab/batch-ci.yml @@ -5,7 +5,11 @@ run go tests batch: USERNAME: fb-idk-access PROJECT: batch_${CI_CONCURRENT_ID} GIT_STRATEGY: clone - stage: test + # this test relies on stuff that happens after build-lattice, which + # makes it pause the entire CI run waiting for this. we accept the + # small risk of wasting a build against the near certainty of spending + # five minutes running only one job. + stage: nonblocking retry: 1 script: - echo "Running test-all"