Try running and building image

2026-09-11 23:31:03 +00:00 · 2023-02-08 15:13:38 -06:00 · 2023-02-08 15:13:38 -06:00 · 3d376b7a1c
commit 3d376b7a1c
parent 126be915a9
2 changed files with 128 additions and 88 deletions
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -29,6 +29,13 @@ stages:
  - nonblocking
  - cleanup_build

+default:
+  image: $CI_IMAGE
+  tags: ["docker+machine"]
+  retry:
+    max: 2
+    when: runner_system_failure
+
 gosec-sast:
  allow_failure: false
  before_script:
@ -60,11 +67,35 @@ gosec-sast:
    paths:
      - .go/pkg/mod/

+.fb-ci_aws_vars:
+  image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
+  variables:
+    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
+    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    AWS_DEFAULT_REGION: "us-east-2"
+
+build_ci_image:
+  stage: ci_image_build
+  image:
+    name: gcr.io/kaniko-project/executor:v1.9.1-debug
+    entrypoint: [""]
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "push"' # must be limited to push since we're using 'changes' below
+      changes:
+        # Have to do both files right now because we use the GOVERSION variable specified in this file to
+        # change the GOVERSION installed in the image we create.
+        - .gitlab/Dockerfile-ci
+        - .gitlab/.gitlab-ci.yml
+  script:
+    - /kaniko/executor
+      --build-arg GOVERSION=$GOVERSION
+      --dockerfile "${CI_PROJECT_DIR}/.gitlab/Dockerfile-ci"
+      --destination "${CI_IMAGE}"
+      --cache=true
+
 smoke build:
-  image: golang:$GOVERSION
  extends: .go-cache
  stage: lint
-  allow_failure: false
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
@ -84,7 +115,6 @@ golangci-lint:

 go mod tidy:
  stage: lint
-  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
@ -94,24 +124,14 @@ go mod tidy:

 build lattice:
  stage: test
-  image: node:14
+  extends: .fb-ci_aws_vars
  variables:
-    AWS_PROFILE: "service-fb-ci"
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
    CI: "false"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  before_script:
-    - curl -sS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-    - unzip -qq awscliv2.zip
-    - ./aws/install
    - aws --version
-    - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $AWS_PROFILE
-    - aws sts get-caller-identity # ensure we have a valid AWS login
+    - aws sts get-caller-identity
  script:
    - cd lattice
    - cache=$(find . -type f -print0 | sort -z | xargs -0 sha1sum | sha1sum | cut -d ' ' -f 1)
@ -147,7 +167,6 @@ build lattice:

 build featurebase:
  stage: test
-  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
@ -157,10 +176,12 @@ build featurebase:
    - go install github.com/rakyll/statik@v0.1.7
    - $GOPATH/bin/statik -src=lattice
    - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-    - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
-    - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64"
-    - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
-    - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          GOOS="${goos}" GOARCH="${goarch}" make build FLAGS="-o featurebase_${goos}_${goarch}"
+        done
+      done
  artifacts:
    paths:
      - featurebase_linux_amd64
@ -170,6 +191,7 @@ build featurebase:
  needs:
    - job: build lattice

+# TODO use kaniko here... (how would the SOURCE_DATE_EPOCH part work?)
 build amd container fb:
  stage: test
  tags:
@ -189,7 +211,6 @@ build amd container fb:

 run jest tests:
  stage: test
-  image: node:14
  variables:
    CI: "true"
  rules:
@ -208,7 +229,6 @@ run jest tests:
 # long time and even now it's pretty slow.
 run go tests race:
  stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
-  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
@ -217,15 +237,12 @@ run go tests race:
    - echo "Running featurebase race tests..."
    - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData|batch|idk|v3/dax/test/dax' | paste -s -d, -)
    - RAMDISK=/mnt/ramdisk go test -race -v -timeout=10m ${PKG_LIST//,/ }
-  tags:
-    - docker

 # We run our base tests against $GOVERSION (a reasonably current version that we trust)
 # and use shardwidth22 for them. This gives us a canary for things breaking for
 # unusual shard widths.
 run go tests:
  stage: test
-  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
@ -236,14 +253,9 @@ run go tests:
  artifacts:
    paths:
      - coverage.out
-  tags:
-    - docker

 run go tests dax/test/dax:
  stage: test
-  image: golang:$GOVERSION
-  tags:
-    - aws
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
@ -400,7 +412,6 @@ upload to sonarcloud:

 package for linux amd64:
  stage: build
-  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
@ -408,8 +419,6 @@ package for linux amd64:
    GOOS: "linux"
    GOARCH: "amd64"
  script:
-    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
-    - apt update && apt install nfpm=2.11.3
    - make package
  artifacts:
    paths:
@ -437,7 +446,6 @@ trigger_m-cloud-images:

 package for linux arm64:
  stage: build
-  image: golang:$GOVERSION
  extends: .go-cache
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
@ -445,14 +453,13 @@ package for linux arm64:
    GOOS: "linux"
    GOARCH: "arm64"
  script:
-    - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
-    - apt update && apt install nfpm=2.11.3
    - make package
  artifacts:
    paths:
      - "*.deb"
      - "*.rpm"

+# TODO: do this with kaniko
 build arm container fb:
  stage: build
  needs:
@ -519,6 +526,7 @@ idk build_arm64:

 # building them all serially because otherwise you get container name conflicts.
 # only do containers on default branch
+# TODO do this with kaniko
 idk package_docker_all:
  stage: build
  tags:
@ -536,21 +544,12 @@ idk package_docker_all:

 idk s3 dump:
  stage: post build
-  allow_failure: false
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-  tags:
-    - shell
+  extends: .fb-ci_aws_vars
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - aws sts get-caller-identity
  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/ --recursive
    - aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/ --recursive
  needs:
@ -559,21 +558,14 @@ idk s3 dump:

 idk s3 dump tag:
  stage: post build
+  extends: .fb-ci_aws_vars
  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
    LOCATION: molecula-artifact-storage/idk/_tags
-  tags:
-    - shell
  rules:
    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+  before_script:
+    - aws sts get-caller-identity # ensure we have valid login
  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
    - |
      for goos in "darwin" "linux"; do
        for goarch in "amd64" "arm64"; do
@ -593,7 +585,6 @@ idk s3 dump tag:

 external lookup tests:
  stage: integration
-  image: golang:$GOVERSION
  extends: .go-cache
  # TODO: no rules here, do we need to add the rules line?
  variables:
@ -610,48 +601,32 @@ external lookup tests:

 s3 dump:
  stage: post build
-  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
-  tags:
-    - shell
+  extends: .fb-ci_aws_vars
  rules:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web"'
+  before_script:
+    - aws sts get-caller-identity # ensure we have a valid AWS login
  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
-    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64
-    - aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64
-    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64
-    - aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64
-    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64
-    - aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64
-    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
-    - aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
+    - |
+      for goos in "darwin" "linux"; do
+        for goarch in "amd64" "arm64"; do
+          aws s3 cp "featurebase_${goos}_${goarch}" "s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_${goos}_${goarch}"
+          aws s3 cp "featurebase_${goos}_${goarch}" "s3://molecula-artifact-storage/featurbase/${CI_COMMIT_BRANCH}/_latest/featurebase_${goos}_${goarch}"
+        done
+      done
  needs:
    - job: build featurebase

 s3 dump tag:
  stage: post build
+  extends: .fb-ci_aws_vars
  variables:
-    PROFILE: "service-fb-ci"
-    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
-    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
-    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
    LOCATION: molecula-artifact-storage/featurebase/_tags
-  tags:
-    - shell
  rules:
    - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
+  before_script:
+    - aws sts get-caller-identity # ensure we have a valid AWS login
  script:
-    - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
-    - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
-    - aws configure set region "us-east-2"
-    - aws configure set aws_profile $PROFILE
    - |
      for goos in "darwin" "linux"; do
        for goarch in "amd64" "arm64"; do
--- a/.gitlab/Dockerfile-ci
+++ b/.gitlab/Dockerfile-ci
@ -0,0 +1,65 @@
+ARG GOVERSION
+FROM golang:$GOVERSION
+
+# not a secure way to do this
+RUN echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
+
+RUN set -eux; \
+  apt-get update; \
+  apt-get install -y --no-install-recommends \
+  nfpm=2.11.3 \
+  unzip \
+  postgresql-client \
+  gnupg \
+  software-properties-common \
+  curl \
+  git \
+  jq \
+  wget \
+  git \
+  libnss3-tools \
+  xz-utils \
+  openssh-client \
+  ; \
+  wget -O- https://apt.releases.hashicorp.com/gpg | \
+  gpg --dearmor | \
+  tee /usr/share/keyrings/hashicorp-archive-keyring.gpg; \
+  echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/hashicorp.list; \
+  apt-get update; \
+  apt-get install \
+  terraform \
+  ; \
+  rm -rf /var/lib/apt/lists/*
+
+RUN set -eux; \
+  curl -sS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"; \
+  unzip -qq awscliv2.zip; \
+  ./aws/install; \
+  rm -rf aws awscliv2.zip
+
+RUN go install github.com/rakyll/statik@v0.1.7
+
+ENV NODE_VERSION 14.21.1
+ENV ARCH x64
+
+# Still need to install nodejs somehow...
+RUN set -eux; \
+  curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz"; \
+  tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner; \
+  rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz"; \
+  ln -s /usr/local/bin/node /usr/local/bin/nodejs; \
+  # smoke tests
+  node --version; \
+  npm --version
+
+ENV YARN_VERSION 1.22.19
+
+RUN set -eux; \
+  curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz"; \
+  mkdir -p /opt; \
+  tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/; \
+  ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn; \
+  ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg; \
+  rm yarn-v$YARN_VERSION.tar.gz; \
+  # smoke test
+  yarn --version