running all of gauntlet in schedule

2026-10-10 12:57:54 +00:00 · 2022-01-05 12:25:41 -06:00 · 2022-01-05 12:25:41 -06:00 · a2ee26b57c
commit a2ee26b57c
parent 03fc5d470d
11 changed files with 1670 additions and 21 deletions
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -26,7 +26,7 @@ golangci-lint:
  extends: .go-cache
  allow_failure: false
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Checking for issues in new code"
    - golangci-lint run -v 
@ -37,7 +37,7 @@ build lattice:
  variables:
    CI: "false"
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - cd lattice
    - yarn install
@ -57,7 +57,7 @@ run jest tests:
  variables:
    CI: "true"
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Testing lattice..."
    - cd lattice
@ -72,7 +72,7 @@ run go tests:
  image: golang:$GOVERSION
  extends: .go-cache
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Running featurebase unit tests..."
    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
@ -86,7 +86,7 @@ run go tests future:
  image: golang:1.17.3
  extends: .go-cache
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Running featurebase unit tests..."
    - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
@ -95,12 +95,11 @@ run go tests future:
    paths:
      - coverage.out

-
 run go tests with output:
  stage: test
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - echo "Running featurebase unit tests to capture JSON output..."
    - go test -json > test-report.out
@ -114,7 +113,7 @@ upload to sonarcloud:
  variables:
    SONAR_TOKEN: $SONAR_TOKEN
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out  -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
  needs:
@ -126,7 +125,7 @@ build for linux amd64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz 
@ -141,7 +140,7 @@ build for linux arm64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz 
@ -156,7 +155,7 @@ build for darwin amd64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz 
@ -171,7 +170,7 @@ build for darwin arm64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - rm -r lattice
    - tar -xvf lattice.tar.gz 
@ -186,7 +185,7 @@ package for linux amd64:
  stage: build
  image: golang:$GOVERSION
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  variables:
    GOOS: "linux"
    GOARCH: "amd64"
@ -208,7 +207,7 @@ build container fb:
  tags:
    - shell
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  before_script:    
    - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
  script:    
@ -226,8 +225,9 @@ smoke test:
    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
    TF_VAR_cluster_prefix: ""
+    TF_VAR_branch: ""
  rules:
-    - if: '$CI_PIPELINE_SOURCE == "push"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  before_script:
    - apt-get update && apt-get install -y gnupg software-properties-common curl git
    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
@ -253,6 +253,8 @@ smoke test:
    - export PATH=$PATH:/usr/local/go/bin
    - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    - TF_VAR_branch=$CI_COMMIT_BRANCH
+    - echo "Branch --> $TF_VAR_branch"
  script:
    - ./qa/scripts/setupSmokeTest.sh
    - ./qa/scripts/testSmokeTest.sh
@ -270,8 +272,12 @@ gauntlet:
    AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
    AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
+    TF_VAR_cluster_prefix: ""
+    TF_VAR_branch: ""
  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
+#    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+#    - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
+    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  before_script:
    - apt-get update && apt-get install -y gnupg software-properties-common curl git
    - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
@ -295,8 +301,12 @@ gauntlet:
    - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
    - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
    - export PATH=$PATH:/usr/local/go/bin
+    - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
+    - echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
+    - TF_VAR_branch=$CI_COMMIT_BRANCH
+    - echo "Branch --> $TF_VAR_branch"
  script:
    - ./qa/scripts/setupSamsungGauntlet.sh
-    - ./qa/scripts/testSamsungGauntlet.sh
+#    - ./qa/scripts/testSamsungGauntlet.sh
  after_script:
    - ./qa/scripts/teardownSamsungGauntlet.sh
--- a/qa/tf/.modules/featurebase-cluster/main.tf
+++ b/qa/tf/.modules/featurebase-cluster/main.tf
@ -46,7 +46,7 @@ resource "aws_instance" "fb_cluster_nodes" {
    Role   = "cluster_node"
  }

-  user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region }))
+  user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region }))
 }

 resource "aws_instance" "fb_ingest" {
@ -79,7 +79,7 @@ resource "aws_instance" "fb_ingest" {
    Role   = "ingest_node"
  }

-  user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region }))
+  user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region }))
 }

 resource "aws_key_pair" "gitlab-featurebase-ci" {
--- a/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl
+++ b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl
@ -164,7 +164,7 @@ write_featurebase_service_file

 #get the featurebase binary and put in in the right spot
 echo "Getting featurebase binary..."
-curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+curl --fail --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
 chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
 chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"

--- a/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl
+++ b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl
@ -48,7 +48,7 @@ get_aws_instance_id
 wait_on_all_ingest_ips

 echo "Getting featurebase binary..."
-curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
+curl --fail --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
 chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
 chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"

--- a/qa/tf/.modules/featurebase-cluster/variables.tf
+++ b/qa/tf/.modules/featurebase-cluster/variables.tf
@ -91,3 +91,8 @@ variable "gitlab_token" {
  description = "Gitlab API token"
  type        = string
 }
+
+variable "branch" {
+  description = "The branch we are on"
+  type        = string
+}
--- a/qa/tf/ci/smoketest/main.tf
+++ b/qa/tf/ci/smoketest/main.tf
@ -7,4 +7,5 @@ module "ci-cluster" {
    fb_data_node_type = "m6g.large"
    fb_data_node_count = 1
    gitlab_token = var.gitlab_token
+    branch = var.branch 
 }
--- a/qa/tf/ci/smoketest/outputs.json
+++ b/qa/tf/ci/smoketest/outputs.json
@ -0,0 +1,26 @@
+{
+  "data_node_ips": {
+    "sensitive": false,
+    "type": [
+      "tuple",
+      [
+        "string"
+      ]
+    ],
+    "value": [
+      "10.0.1.135"
+    ]
+  },
+  "ingest_ips": {
+    "sensitive": false,
+    "type": [
+      "tuple",
+      [
+        "string"
+      ]
+    ],
+    "value": [
+      "3.128.205.2"
+    ]
+  }
+}
--- a/qa/tf/ci/smoketest/terraform.tfstate.backup
+++ b/qa/tf/ci/smoketest/terraform.tfstate.backup
--- a/qa/tf/ci/smoketest/variables.tf
+++ b/qa/tf/ci/smoketest/variables.tf
@ -16,4 +16,9 @@ variable "gitlab_token" {
 variable "cluster_prefix" {
  type        = string
  description = "This is a identifier that will be prefixed to created resources"
+}
+
+variable "branch" {
+  type        = string
+  description = "The branch we are on"
 }
--- a/qa/tf/gauntlet/samsung/main.tf
+++ b/qa/tf/gauntlet/samsung/main.tf
@ -10,4 +10,5 @@ module "samsung-cluster" {
    fb_ingest_disk_iops = 10000
    fb_ingest_node_count = 1
    gitlab_token = var.gitlab_token
+    branch = var.branch
 }
--- a/qa/tf/gauntlet/samsung/variables.tf
+++ b/qa/tf/gauntlet/samsung/variables.tf
@ -16,4 +16,9 @@ variable "gitlab_token" {
 variable "cluster_prefix" {
  type        = string
  description = "This is a identifier that will be prefixed to created resources"
+}
+
+variable "branch" {
+  type        = string
+  description = "The branch we are on"
 }