Add GitLab CI pipeline

.gitlab/.gitlab-ci.yml

@@ -0,0 +1,670 @@
+# You will see a couple of instances of:
+# PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
+# This gets us a package list, comma-separated, which excludes the batch
+# and IDK tests, and a couple of subdirs with specialized stuff. We can
+# then use this with -coverpkg, or we can use ${PKG_LIST//,/ } to get a
+# space-separated list for use with `go test` to run the tests for those
+# directories only.
+
+include:
+  - local: /.gitlab/batch-ci.yml
+  - template: Security/SAST.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml
+
+variables:
+  GOVERSION: "1.19.3"
+  GOFUTURE: "latest"
+  CI_IMAGE: "${CI_REGISTRY_IMAGE}/ci-builder:0.0.1"
+
+stages:
+  - ci_image_build
+  - lint
+  - test
+  - build
+  - post build
+  - integration
+  - gauntlet
+  - performance
+  - nonblocking
+  - cleanup_build
+
+gosec-sast:
+  allow_failure: false
+  before_script:
+    - export GOPRIVATE=github.com/molecula/*
+    - apk add openssh-client
+    - eval $(ssh-agent -s)
+    - echo "$FB_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"
+    - ssh-keygen -F github.com || echo "$SSH_KNOWN_HOSTS_HASHED" >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+  script:
+    ## securego/gosec works for scanning, but not converting to the gitlab report format.
+    - go install github.com/securego/gosec/v2/cmd/gosec@v2.12.0
+    - gosec -fmt=json -out=gosec.json -tests ./... || true
+    ## gitlab's wrapper for gosec works for converting, but not for scanning.
+    - go install 'gitlab.com/gitlab-org/security-products/analyzers/gosec@v1.4.0'
+    - gosec convert gosec.json > gl-sast-report.json
+
+.go-cache:
+  variables:
+    GOPATH: $CI_PROJECT_DIR/.go
+  before_script:
+    - mkdir -p .go
+  cache:
+    # this caching strategy makes it so each branch uses the same cache
+    key: "$CI_COMMIT_REF_SLUG"
+    paths:
+      - .go/pkg/mod/
+
+smoke build:
+  image: golang:$GOVERSION
+  extends: .go-cache
+  stage: lint
+  allow_failure: false
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
+    - go build ./...
+
+golangci-lint:
+  image: golangci/golangci-lint:v1.46.2
+  extends: .go-cache
+  stage: lint
+  allow_failure: true
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Checking for issues in new code"
+    - golangci-lint run -v --timeout=8m
+
+go mod tidy:
+  stage: lint
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - go mod tidy
+    - git diff --exit-code -- go.mod go.sum
+
+build lattice:
+  stage: test
+  image: node:14
+  variables:
+    AWS_PROFILE: "service-fb-ci"
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    CI: "false"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - curl -sS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+    - unzip -qq awscliv2.zip
+    - ./aws/install
+    - aws --version
+    - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $AWS_PROFILE
+    - aws sts get-caller-identity # ensure we have a valid AWS login
+  script:
+    - cd lattice
+    - cache=$(find . -type f -print0 | sort -z | xargs -0 sha1sum | sha1sum | cut -d ' ' -f 1)
+    - echo "'$cache'"
+    - echo "looking for s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
+    # if is for if we had a cache object in S3
+    # else is for if we didn't have a cache object (and have to build).
+    - |
+      if aws s3api head-object --bucket molecula-artifact-storage --key "lattice/$cache/build.tar.gz"; then
+        # download object, extract, name the folder `build`
+        aws s3 cp "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz" build.tar.gz
+        tar -xf build.tar.gz
+      else # cache file not found
+        yarn install --frozen-lockfile # CI needs to enforce that the lockfile doesn't need to be updated
+        yarn build
+        tar -czvf "$cache.tar.gz" build/
+        aws s3 mv "$cache.tar.gz" "s3://molecula-artifact-storage/lattice/$cache/build.tar.gz"
+        touch "$CI_COMMIT_SHA"
+        aws s3 mv "$CI_COMMIT_SHA" "s3://molecula-artifact-storage/lattice/$cache/$CI_COMMIT_SHA"
+      fi
+    - | # Ensure that we have build directory after the caching step
+      if [ ! -d build ]; then
+        echo "no build directory, erroring out" || exit 1
+      fi
+    - mv build ../
+    - cd ../
+    - rm -r lattice
+    - mv build lattice
+    - tar -czvf lattice.tar.gz lattice
+  artifacts:
+    paths:
+      - lattice.tar.gz
+
+build featurebase:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - rm -r lattice
+    - tar -xvf lattice.tar.gz
+    - go install github.com/rakyll/statik@v0.1.7
+    - $GOPATH/bin/statik -src=lattice
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
+    - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64"
+    - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
+    - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
+  artifacts:
+    paths:
+      - featurebase_linux_amd64
+      - featurebase_linux_arm64
+      - featurebase_darwin_amd64
+      - featurebase_darwin_arm64
+  needs:
+    - job: build lattice
+
+build amd container fb:
+  stage: test
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+  script:
+    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
+    - docker push $tag
+    - echo Created docker featurebase image with tag "$tag"
+  needs:
+    - job: build featurebase
+
+run jest tests:
+  stage: test
+  image: node:14
+  variables:
+    CI: "true"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Testing lattice..."
+    - cd lattice
+    - npm install --force
+    - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter
+  artifacts:
+    paths:
+      - lattice/coverage/lcov.info
+
+# We run go test -race on all the standard packages, skipping the ones that have their
+# own separate tests. we spin this off as nonblocking because it used to take a really
+# long time and even now it's pretty slow.
+run go tests race:
+  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
+  script:
+    - echo "Running featurebase race tests..."
+    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
+    - RAMDISK=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ }
+  tags:
+    - docker
+
+# We run our base tests against $GOVERSION (a reasonably current version that we trust)
+# and use shardwidth22 for them. This gives us a canary for things breaking for
+# unusual shard widths.
+run go tests:
+  stage: test
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Running featurebase unit tests..."
+    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
+    - RAMDISK=/mnt/ramdisk go test -tags=shardwidth22 -timeout=10m -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ${PKG_LIST//,/ }
+  artifacts:
+    paths:
+      - coverage.out
+  tags:
+    - docker
+
+run go tests dax/test/dax:
+  stage: test
+  image: golang:$GOVERSION
+  tags:
+    - aws
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - echo "Building FB and Datagen docker images for DAX tests"
+    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
+    - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax
+  artifacts:
+    paths:
+      - coverage-dax-integration.out
+
+.idk_changed:
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+      changes:
+        compare_to: refs/heads/master
+        paths:
+          - idk/**
+          - client/**
+          - batch/**
+
+# idk tests
+run go tests idk race:
+  variables:
+    PROJECT: race_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  rules:
+    - !reference [.idk_changed, rules]
+  script:
+    - echo "Running test-all-race"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all-race
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb
+
+run go tests idk shard transactional:
+  variables:
+    IDK_DEFAULT_SHARD_TRANSACTIONAL: 1
+    PROJECT: shardttrans_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  rules:
+    - !reference [.idk_changed, rules]
+  script:
+    - echo "Running shard transactional tests"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+      - ./idk/testdata/*_logs.txt
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb
+
+run go tests idk 533:
+  variables:
+    USERNAME: fb-idk-access
+    PROJECT: test533_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running confluent 5.3.3 test-all"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  rules:
+    - !reference [.idk_changed, rules]
+  tags:
+    - shell
+    - aws
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+  needs:
+    - job: build amd container fb
+
+run go tests idk sasl:
+  variables:
+    PROJECT: sasl_${CI_CONCURRENT_ID}
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running test-all-kafka-sasl"
+    - cd ./idk/
+    - echo $PROJECT
+    - echo $CI_COMMIT_REF_SLUG
+    - BRANCH_NAME=${CI_COMMIT_REF_SLUG} make test-all-kafka-sasl
+  after_script:
+    - cd ./idk/
+    - make save-pilosa-logs
+    - make shutdown
+  rules:
+    - !reference [.idk_changed, rules]
+  tags:
+    - shell
+    - aws
+  artifacts:
+    paths:
+      - ./idk/testdata/*_coverage.out
+  needs:
+    - job: build amd container fb
+
+upload to sonarcloud:
+  stage: nonblocking
+  image: sonarsource/sonar-scanner-cli:4.7
+  variables:
+    SONAR_TOKEN: $SONAR_TOKEN
+  rules:
+    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out,idk/testdata/*coverage.out,batch/testdata/*coverage.out,coverage-from-docker/*.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
+  needs:
+    - job: run go tests
+    - job: run jest tests
+    - job: external lookup tests
+    - job: run go tests idk race
+      optional: true
+    - job: run go tests idk shard transactional
+      optional: true
+    - job: run go tests idk sasl
+      optional: true
+    - job: run go tests idk 533
+      optional: true
+    - job: run go tests batch
+      optional: true
+    - job: run go tests dax/test/dax
+      optional: true
+
+package for linux amd64:
+  stage: build
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  variables:
+    GOOS: "linux"
+    GOARCH: "amd64"
+  script:
+    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+    - apt update && apt install nfpm=2.11.3
+    - make package
+  artifacts:
+    paths:
+      - "*.deb"
+      - "*.rpm"
+
+upload_artifacts_to_nexus:
+  stage: post build
+  image: golang:$GOVERSION
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  script:
+    - find . -maxdepth 1 -name '*.rpm' -exec curl -v --user "$NEXUS_YUM_CREDS" --upload-file {} https://nexus.molecula.com/repository/molecula-yum/release/ \;
+  dependencies:
+    - package for linux amd64
+    - package for linux arm64
+
+trigger_m-cloud-images:
+  stage: post build
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  needs:
+    - upload_artifacts_to_nexus
+  trigger: molecula/m-cloud-images
+
+package for linux arm64:
+  stage: build
+  image: golang:$GOVERSION
+  extends: .go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  variables:
+    GOOS: "linux"
+    GOARCH: "arm64"
+  script:
+    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+    - apt update && apt install nfpm=2.11.3
+    - make package
+  artifacts:
+    paths:
+      - "*.deb"
+      - "*.rpm"
+
+build arm container fb:
+  stage: build
+  needs:
+    - "build featurebase"
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
+  script:
+    - echo $CI_COMMIT_REF_SLUG
+    - tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 --build-arg SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH -t $tag -f .gitlab/Dockerfile .
+    - docker push $tag
+    - echo Created docker featurebase image with tag "$tag"
+
+### start idk builds ###
+# building them all serially because otherwise you get container name conflicts.
+idk build_amd64:
+  stage: build
+  variables:
+    BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - cd ./idk/
+    - date
+    - make docker-build GOOS="linux" GOARCH="amd64" BUILD_CGO=1
+    - date
+    - make docker-build GOOS="darwin" GOARCH="amd64"
+    - date
+  artifacts:
+    paths:
+      - ./idk/build/*
+  needs:
+    # doesn't actually need this... just want it to start executing before *all* the tests finish
+    - job: run go tests
+
+idk build_arm64:
+  stage: build
+  tags:
+    - shell-arm64
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - cd ./idk/
+    - date
+    - make docker-build GOOS="linux" GOARCH="arm64" BUILD_CGO=1 BUILD_NAME="linux-arm64"
+    - date
+    - make docker-build GOOS="darwin" GOARCH="arm64"
+    - date
+  artifacts:
+    paths:
+      - ./idk/build/*
+  needs:
+    # doesn't actually need this... just want it to start executing before *all* the tests finish
+    - job: run go tests
+
+# building them all serially because otherwise you get container name conflicts.
+# only do containers on default branch
+idk package_docker_all:
+  stage: build
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+    - make docker-idk GOOS="linux" GOARCH="amd64"
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - make docker-idk-tag-push GOOS="linux" GOARCH="amd64"
+  needs:
+    - job: idk build_amd64
+    - job: idk build_arm64
+
+idk s3 dump:
+  stage: post build
+  allow_failure: false
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/ --recursive
+    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/ --recursive
+  needs:
+    - job: idk build_amd64
+    - job: idk build_arm64
+
+idk s3 dump tag:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    LOCATION: molecula-artifact-storage/idk/_tags
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          dir=idk-${CI_COMMIT_TAG}-${goos}-${goarch}
+          echo "Directory ${dir}"
+          mkdir ${dir}
+          mv ./idk/build/idk-${goos}-${goarch}/molecula-consumer-* ${dir}/
+          tar cvzf ${dir}.tar.gz ${dir}
+          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
+          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
+        done
+      done
+  needs:
+    - job: idk build_amd64
+    - job: idk build_arm64
+### end idk builds ###
+
+external lookup tests:
+  stage: integration
+  image: golang:$GOVERSION
+  extends: .go-cache
+  # TODO: no rules here, do we need to add the rules line?
+  variables:
+    POSTGRES_DB: $POSTGRES_DB
+    POSTGRES_USER: $POSTGRES_USER
+    POSTGRES_PASSWORD: $POSTGRES_PASSWORD
+    POSTGRES_HOST_AUTH_METHOD: trust
+  services:
+    - postgres:13.5
+  script:
+    - apt-get update --allow-releaseinfo-change -y
+    - apt-get install -y postgresql-client
+    - go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
+
+s3 dump:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+  tags:
+    - shell
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web"'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64
+    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64
+    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64
+    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64
+    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64
+    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64
+    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
+    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
+  needs:
+    - job: build featurebase
+
+s3 dump tag:
+  stage: post build
+  variables:
+    PROFILE: "service-fb-ci"
+    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    LOCATION: molecula-artifact-storage/featurebase/_tags
+  tags:
+    - shell
+  rules:
+    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
+  script:
+    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
+    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
+    - aws configure set region "us-east-2"
+    - aws configure set aws_profile $PROFILE
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch}
+          echo "Directory ${dir}"
+          mkdir $dir
+          mv featurebase_${goos}_${goarch} ${dir}/featurebase
+          cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
+          tar cvzf ${dir}.tar.gz ${dir}
+          aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
+          aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
+        done
+      done
+
+  needs:
+    - job: build featurebase

.gitlab/Dockerfile

@@ -0,0 +1,24 @@
+FROM alpine:3.14.2
+
+LABEL maintainer "dev@molecula.com"
+LABEL org.opencontainers.image.authors="dev@molecula.com"
+
+ARG ARCH
+
+WORKDIR /
+
+RUN apk add --no-cache curl jq
+
+COPY NOTICE .
+COPY featurebase_linux_$ARCH featurebase
+
+RUN chmod ugo+x .
+EXPOSE 10101
+VOLUME /data
+
+ENV PILOSA_DATA_DIR /data
+ENV PILOSA_BIND 0.0.0.0:10101
+ENV PILOSA_BIND_GRPC 0.0.0.0:20101
+
+ENTRYPOINT ["/featurebase"]
+CMD ["server"]

.gitlab/batch-ci.yml

@@ -0,0 +1,28 @@
+run go tests batch:
+  variables:
+    PROJECT: batch_${CI_CONCURRENT_ID}
+  # this test relies on stuff that happens after build-lattice, which
+  # makes it pause the entire CI run waiting for this. we accept the
+  # small risk of wasting a build against the near certainty of spending
+  # five minutes running only one job.
+  stage: nonblocking
+  retry: 1
+  script:
+    - echo "Running test-all"
+    - cd ./batch/
+    - echo $PROJECT
+    - make build-featurebase
+    - make test-all
+  after_script:
+    - cd ./batch/
+    - make save-featurebase-logs
+    - make shutdown
+  artifacts:
+    paths:
+      - ./batch/testdata/*_coverage.out
+      - ./batch/testdata/*_logs.txt
+  tags:
+    - shell
+    - aws
+  needs:
+    - job: build amd container fb