shuffle some CI stages, improve IDK test determination

mark IDK tests and batch tests as "nonblocking" so we don't wait
on them before doing builds. this risks running builds when the
IDK or batch tests could fail. It also unrisks spending an
extra five minutes in CI waiting for the last "test" phase thing
to run in a long dependency chain, which is potentially more
significant.

Make IDK tests run for changes to idk, client, or batch directories,
because at least one IDK thing uses the batch importer. Also,
make that actually work, we think -- verified that the IDK tests
got run when there were changes in that directory, but there
aren't any now, so we don't expect to see them.

Also, move the "IDK changed" rules into their own heading and
incorporate them by reference, and simplify the conditional because
it seemed to be Acting Up, but also make it check against
refs/heads/master, rather than possibly just the parent commit,
since that seems to be more consistent.

Finally, we combine four of the tests (go tests, go tests future,
go tests future plg, go tests shardwidth22) into two tests, both
using the "future" compiler, one for plg, one for shardwidth22,
so we don't need as many parallel runners and are less likely
to end up waiting on them.

At this point, the largest delay in CI is the chain through
building lattice, which blocks some tasks for a fairly long
time because they just have to wait for us to have built a
container we can use as the server container in tests that need
a server to work against.

.gitlab/.gitlab-ci.yml

@@ -4,6 +4,16 @@ include:
   - template: Security/License-Scanning.gitlab-ci.yml
   - template: Security/Dependency-Scanning.gitlab-ci.yml
 
+.idk_changed:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+      changes:
+        compare_to: refs/heads/master
+        paths:
+          - idk/**
+          - client/**
+          - batch/**
+
 gosec-sast:
   allow_failure: false
   before_script:
@@ -189,19 +199,6 @@ run jest tests:
     paths:
       - lattice/coverage/lcov.info
 
-run go tests:
-  stage: test
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  retry: 1
-  script:
-    - echo "Running featurebase unit tests..."
-    - go test -v -timeout=10m $(go list ./... | grep -Ev 'batch|idk')
-  tags:
-    - aws
-
 run go tests race:
   stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
   image: golang:$GOVERSION
@@ -216,22 +213,10 @@ run go tests race:
   tags:
     - aws
 
-run go tests shardwidth22:
-  stage: test
-  image: golang:$GOVERSION
-  extends: .go-cache
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
-  script:
-    - echo "Running featurebase shardwidth22 tests..."
-    - go test -timeout=10m -tags=shardwidth22 -v $(go list ./... | grep -Ev 'batch|idk')
-  tags:
-    - aws
-
-# we do coverage reporting from the future tests because the json
-# output is very difficult to human-read. The alternative would be to
-# run the regular tests twice and also run the future tests.
-run go tests future:
+# we dump output in json mode because we're sending the test report data to sonarcloud. it's
+# harder to read but easier to use, and running it an extra time without json, when we expect
+# tests to succeed, makes CI slow
+run go tests:
   stage: test
   image: golang:$GOFUTURE
   extends: .go-cache
@@ -241,7 +226,7 @@ run go tests future:
   script:
     - echo "Running featurebase unit tests..."
     - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | grep -Ev 'batch|idk' | paste -s -d, -)
-    - go test -timeout=10m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev 'batch|idk') | tee test-report.out
+    - go test -timeout=10m -json -tags=shardwidth22 -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev 'batch|idk') | tee test-report.out
   artifacts:
     paths:
       - coverage.out
@@ -249,7 +234,8 @@ run go tests future:
   tags:
     - aws
 
-run go tests future plg:
+# json mode because test report data goes to sonarcloud
+run go tests plg:
   stage: test
   image: golang:$GOFUTURE
   extends: .go-cache
@@ -274,13 +260,10 @@ run go tests idk race:
   variables:
     USERNAME: fb-idk-access
     PROJECT: race_${CI_CONCURRENT_ID}
-  stage: test
+  stage: nonblocking
   retry: 1
   rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-      changes:
-        - idk/**/*
-        - client/**/*
+    - !reference [.idk_changed, rules]
   script:
     - echo "Running test-all-race"
     - cd ./idk/
@@ -309,13 +292,10 @@ run go tests idk shard transactional:
     IDK_DEFAULT_SHARD_TRANSACTIONAL: 1
     USERNAME: fb-idk-access
     PROJECT: shardttrans_${CI_CONCURRENT_ID}
-  stage: test
+  stage: nonblocking
   retry: 1
   rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-      changes:
-        - idk/**/*
-        - client/**/*
+    - !reference [.idk_changed, rules]
   script:
     - echo "Running shard transactional tests"
     - cd ./idk/
@@ -344,7 +324,7 @@ run go tests idk 533:
   variables:
     USERNAME: fb-idk-access
     PROJECT: test533_${CI_CONCURRENT_ID}
-  stage: test
+  stage: nonblocking
   retry: 1
   script:
     - echo "Running confluent 5.3.3 test-all"
@@ -357,10 +337,7 @@ run go tests idk 533:
     - make save-pilosa-logs
     - make shutdown
   rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-      changes:
-        - idk/**/*
-        - client/**/*
+    - !reference [.idk_changed, rules]
   tags:
     - shell
     - aws
@@ -377,7 +354,7 @@ run go tests idk sasl:
   variables:
     USERNAME: fb-idk-access
     PROJECT: sasl_${CI_CONCURRENT_ID}
-  stage: test
+  stage: nonblocking
   retry: 1
   script:
     - echo "Running test-all-kafka-sasl"
@@ -390,10 +367,7 @@ run go tests idk sasl:
     - make save-pilosa-logs
     - make shutdown
   rules:
-    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
-      changes:
-        - idk/**/*
-        - client/**/*
+    - !reference [.idk_changed, rules]
   tags:
     - shell
     - aws
@@ -406,7 +380,7 @@ run go tests idk sasl:
 
 
 upload to sonarcloud:
-  stage: integration
+  stage: nonblocking
   image: sonarsource/sonar-scanner-cli:4.6
   variables:
     SONAR_TOKEN: $SONAR_TOKEN
@@ -415,8 +389,8 @@ upload to sonarcloud:
   script:
     - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out,batch/testdata/*coverage.out -Dsonar.go.tests.reportPaths=test-report*.out,idk/testdata/*report.out,batch/testdata/*report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
   needs:
-    - job: run go tests future plg
-    - job: run go tests future
+    - job: run go tests
+    - job: run go tests plg
     - job: run jest tests
     - job: external lookup tests
     - job: run go tests idk race

.gitlab/batch-ci.yml

@@ -5,7 +5,11 @@ run go tests batch:
     USERNAME: fb-idk-access
     PROJECT: batch_${CI_CONCURRENT_ID}
     GIT_STRATEGY: clone
-  stage: test
+  # this test relies on stuff that happens after build-lattice, which
+  # makes it pause the entire CI run waiting for this. we accept the
+  # small risk of wasting a build against the near certainty of spending
+  # five minutes running only one job.
+  stage: nonblocking
   retry: 1
   script:
     - echo "Running test-all"