diff --git a/.artifactory/pipelines.yml b/.artifactory/pipelines.yml deleted file mode 100644 index 3e8890192..000000000 --- a/.artifactory/pipelines.yml +++ /dev/null @@ -1,38 +0,0 @@ - -resources: - - name: featurebaseRepo - type: GitRepo - configuration: - # SCM integration where the repository is located - gitProvider: github_molecula_featurebase - # Repository path, including org name/repo name - path: molecula/featurebase - branches: - # Specifies which branches will trigger dependent steps - include: cicd - - name: featurebaseBuildInfo - type: BuildInfo - configuration: - sourceArtifactory: Molecula_Artifactory - buildName: featurebase_build - buildNumber: 4 -pipelines: - - name: ScanGoCode - steps: - - name: scan - type: XrayScan - configuration: - failOnScan: false - inputResources: - - name: featurebaseBuildInfo - trigger: true - execution: - onStart: - - echo "Preparing for work..." - - echo "Prepping build environment" - onSuccess: - - echo "Job well done!" - onFailure: - - echo "uh oh, something went wrong" - onComplete: - - echo "Cleaning up some stuff" \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml index c3e1a43e4..ce81cb4af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,305 +1,305 @@ -version: 2.1 +# version: 2.1 -executors: - golang: - parameters: - version: - type: string - default: "1.15.8" - resource_class: - type: string - default: medium - docker: - - image: circleci/golang:<< parameters.version >> - resource_class: << parameters.resource_class >> - working_directory: /go/src/github.com/molecula/featurebase +# executors: +# golang: +# parameters: +# version: +# type: string +# default: "1.15.8" +# resource_class: +# type: string +# default: medium +# docker: +# - image: circleci/golang:<< parameters.version >> +# resource_class: << parameters.resource_class >> +# working_directory: /go/src/github.com/molecula/featurebase -commands: - add-github-auth: - steps: - - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/" - - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:" - restore-mod-cache: - steps: - - restore_cache: - key: mod-cache-{{ checksum "go.sum" }} - save-mod-cache: - steps: - - save_cache: - key: mod-cache-{{ checksum "go.sum" }} - paths: - - /go/pkg/mod/ - checkout-plus: - steps: - - add-github-auth - - checkout - - restore-mod-cache - skip-if-root-unchanged: - description: "skips the parent job if the PR includes no changes to featurebase" - steps: - - run: | - ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true - echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES" - if [ -z "$ROOT_CHANGED_FILES" ] ; then - echo "halting step" - circleci step halt - fi - skip-if-lattice-unchanged: - description: "skips the parent job if the PR includes no changes to lattice" - steps: - - run: | - LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true - echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES" - if [ -z "$LATTICE_CHANGED_FILES" ] ; then - echo "halting step" - circleci step halt - fi +# commands: +# add-github-auth: +# steps: +# - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/" +# - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:" +# restore-mod-cache: +# steps: +# - restore_cache: +# key: mod-cache-{{ checksum "go.sum" }} +# save-mod-cache: +# steps: +# - save_cache: +# key: mod-cache-{{ checksum "go.sum" }} +# paths: +# - /go/pkg/mod/ +# checkout-plus: +# steps: +# - add-github-auth +# - checkout +# - restore-mod-cache +# skip-if-root-unchanged: +# description: "skips the parent job if the PR includes no changes to featurebase" +# steps: +# - run: | +# ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true +# echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES" +# if [ -z "$ROOT_CHANGED_FILES" ] ; then +# echo "halting step" +# circleci step halt +# fi +# skip-if-lattice-unchanged: +# description: "skips the parent job if the PR includes no changes to lattice" +# steps: +# - run: | +# LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true +# echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES" +# if [ -z "$LATTICE_CHANGED_FILES" ] ; then +# echo "halting step" +# circleci step halt +# fi -jobs: - setup: - executor: - name: golang - steps: - - checkout-plus - - run: go mod download - - save-mod-cache - linter: - executor: - name: golang - steps: - - checkout-plus - - skip-if-root-unchanged - - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0 - - run: make golangci-lint - go-mod-tidy: - executor: - name: golang - steps: - - checkout-plus - - skip-if-root-unchanged - - run: go mod tidy - - run: git diff --exit-code -- go.mod go.sum - check-changelog-label: - executor: - name: golang - steps: - - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request - - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e - test-build-arm: - executor: - name: golang - steps: - - checkout-plus - - skip-if-root-unchanged - - run: make build GOOS=linux GOARCH=arm GOARM=5 - - run: make build GOOS=linux GOARCH=arm GOARM=6 - - run: make build GOOS=linux GOARCH=arm GOARM=7 - - run: make build GOOS=linux GOARCH=arm64 - test: - parameters: - resource_class: - type: string - default: medium - golang_version: - type: string - default: "1.15.8" - shard_width: - type: string - default: "20" - test_make_target: - type: string - default: "test" - test_flags: - type: string - default: "" - goarch: - type: string - default: amd64 - executor: - name: golang - version: << parameters.golang_version >> - resource_class: << parameters.resource_class >> - environment: - TMPDIR: /mnt/ramdisk - steps: - - checkout-plus - - skip-if-root-unchanged - - run: sudo apt-get update --allow-releaseinfo-change -y - - run: sudo apt-get install lsof - - run: - command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >> - no_output_timeout: 30m - test-external-lookup: - docker: - - image: circleci/golang:1.15.8 - - image: circleci/postgres:13.2-ram - environment: - POSTGRES_PASSWORD=password - steps: - - checkout-plus - - skip-if-root-unchanged - - run: sudo apt-get update --allow-releaseinfo-change -y - - run: sudo apt-get install postgresql-client - - run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1) - - run: - command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable - no_output_timeout: 30m - cluster-tests: - executor: - name: golang - steps: - - checkout-plus - - skip-if-root-unchanged - - setup_remote_docker - - run: make clustertests-build - release: - executor: - name: golang - steps: - - checkout-plus - - attach_workspace: - at: . - - setup_remote_docker: - version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 - - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - - run: make docker-release - - store_artifacts: - path: build - - persist_to_workspace: - root: . - paths: build - publish_release: - executor: - name: golang - steps: - - attach_workspace: - at: . - - run: go get github.com/tcnksm/ghr - - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/ - docker-build: - executor: - name: golang - steps: - - checkout-plus - - setup_remote_docker: - version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 - - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - - run: make docker GO_VERSION=1.15.8 - - run: docker run featurebase:$(git describe --tags) help - dockerhub-upload-unstable: - executor: - name: golang - steps: - - checkout-plus - - setup_remote_docker: - version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 - - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - - run: make docker - - run: docker run featurebase:$(git describe --tags) help - - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >> - dockerhub-upload-stable: - executor: - name: golang - steps: - - checkout-plus - - setup_remote_docker: - version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 - - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - - run: make docker - - run: docker run featurebase:$(git describe --tags) help - - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >> - - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest +# jobs: +# setup: +# executor: +# name: golang +# steps: +# - checkout-plus +# - run: go mod download +# - save-mod-cache +# linter: +# executor: +# name: golang +# steps: +# - checkout-plus +# - skip-if-root-unchanged +# - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0 +# - run: make golangci-lint +# go-mod-tidy: +# executor: +# name: golang +# steps: +# - checkout-plus +# - skip-if-root-unchanged +# - run: go mod tidy +# - run: git diff --exit-code -- go.mod go.sum +# check-changelog-label: +# executor: +# name: golang +# steps: +# - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request +# - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e +# test-build-arm: +# executor: +# name: golang +# steps: +# - checkout-plus +# - skip-if-root-unchanged +# - run: make build GOOS=linux GOARCH=arm GOARM=5 +# - run: make build GOOS=linux GOARCH=arm GOARM=6 +# - run: make build GOOS=linux GOARCH=arm GOARM=7 +# - run: make build GOOS=linux GOARCH=arm64 +# test: +# parameters: +# resource_class: +# type: string +# default: medium +# golang_version: +# type: string +# default: "1.15.8" +# shard_width: +# type: string +# default: "20" +# test_make_target: +# type: string +# default: "test" +# test_flags: +# type: string +# default: "" +# goarch: +# type: string +# default: amd64 +# executor: +# name: golang +# version: << parameters.golang_version >> +# resource_class: << parameters.resource_class >> +# environment: +# TMPDIR: /mnt/ramdisk +# steps: +# - checkout-plus +# - skip-if-root-unchanged +# - run: sudo apt-get update --allow-releaseinfo-change -y +# - run: sudo apt-get install lsof +# - run: +# command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >> +# no_output_timeout: 30m +# test-external-lookup: +# docker: +# - image: circleci/golang:1.15.8 +# - image: circleci/postgres:13.2-ram +# environment: +# POSTGRES_PASSWORD=password +# steps: +# - checkout-plus +# - skip-if-root-unchanged +# - run: sudo apt-get update --allow-releaseinfo-change -y +# - run: sudo apt-get install postgresql-client +# - run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1) +# - run: +# command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable +# no_output_timeout: 30m +# cluster-tests: +# executor: +# name: golang +# steps: +# - checkout-plus +# - skip-if-root-unchanged +# - setup_remote_docker +# - run: make clustertests +# release: +# executor: +# name: golang +# steps: +# - checkout-plus +# - attach_workspace: +# at: . +# - setup_remote_docker: +# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 +# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin +# - run: make docker-release +# - store_artifacts: +# path: build +# - persist_to_workspace: +# root: . +# paths: build +# publish_release: +# executor: +# name: golang +# steps: +# - attach_workspace: +# at: . +# - run: go get github.com/tcnksm/ghr +# - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/ +# docker-build: +# executor: +# name: golang +# steps: +# - checkout-plus +# - setup_remote_docker: +# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 +# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin +# - run: make docker GO_VERSION=1.15.8 +# - run: docker run featurebase:$(git describe --tags) help +# dockerhub-upload-unstable: +# executor: +# name: golang +# steps: +# - checkout-plus +# - setup_remote_docker: +# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 +# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin +# - run: make docker +# - run: docker run featurebase:$(git describe --tags) help +# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >> +# dockerhub-upload-stable: +# executor: +# name: golang +# steps: +# - checkout-plus +# - setup_remote_docker: +# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 +# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin +# - run: make docker +# - run: docker run featurebase:$(git describe --tags) help +# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >> +# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest -workflows: - build: - jobs: - - setup: - context: molecula - filters: - tags: - only: /^v.*/ - - linter: - context: molecula - requires: - - setup - - go-mod-tidy: - context: molecula - requires: - - setup - - check-changelog-label: - context: molecula - requires: - - setup - - test-build-arm: - context: molecula - requires: - - setup - - test: - name: test-golang-<< matrix.golang_version >> - resource_class: large - context: molecula - requires: - - setup - matrix: - parameters: - golang_version: ["1.15.8", "1.16.10"] - - test: - name: << matrix.test_make_target >> - resource_class: xlarge - context: molecula - requires: - - setup - matrix: - parameters: - test_make_target: ["test-race"] - - test: - name: test-shardwidth-22 - context: molecula - shard_width: "22" - resource_class: large - requires: - - setup - - test-external-lookup: - context: molecula - requires: - - setup - - cluster-tests: - context: molecula - requires: - - setup - - docker-build: - context: molecula - requires: - - setup - - release: - context: molecula - requires: - - setup - filters: - tags: - only: /^v.*/ - - publish_release: - context: molecula - requires: - - release - filters: - tags: - only: /^v.*/ - branches: - ignore: /.*/ - - dockerhub-upload-unstable: - context: molecula - requires: - - setup - filters: - branches: - only: master - - dockerhub-upload-stable: - context: molecula - requires: - - setup - filters: - tags: - only: /^v.*/ - branches: - ignore: /.*/ +# workflows: +# build: +# jobs: +# - setup: +# context: molecula +# filters: +# tags: +# only: /^v.*/ +# - linter: +# context: molecula +# requires: +# - setup +# - go-mod-tidy: +# context: molecula +# requires: +# - setup +# - check-changelog-label: +# context: molecula +# requires: +# - setup +# - test-build-arm: +# context: molecula +# requires: +# - setup +# - test: +# name: test-golang-<< matrix.golang_version >> +# resource_class: large +# context: molecula +# requires: +# - setup +# matrix: +# parameters: +# golang_version: ["1.15.8", "1.16.10"] +# - test: +# name: << matrix.test_make_target >> +# resource_class: xlarge +# context: molecula +# requires: +# - setup +# matrix: +# parameters: +# test_make_target: ["test-race"] +# - test: +# name: test-shardwidth-22 +# context: molecula +# shard_width: "22" +# resource_class: large +# requires: +# - setup +# - test-external-lookup: +# context: molecula +# requires: +# - setup +# - cluster-tests: +# context: molecula +# requires: +# - setup +# - docker-build: +# context: molecula +# requires: +# - setup +# - release: +# context: molecula +# requires: +# - setup +# filters: +# tags: +# only: /^v.*/ +# - publish_release: +# context: molecula +# requires: +# - release +# filters: +# tags: +# only: /^v.*/ +# branches: +# ignore: /.*/ +# - dockerhub-upload-unstable: +# context: molecula +# requires: +# - setup +# filters: +# branches: +# only: master +# - dockerhub-upload-stable: +# context: molecula +# requires: +# - setup +# filters: +# tags: +# only: /^v.*/ +# branches: +# ignore: /.*/ diff --git a/.gitignore b/.gitignore index 2082bd5c0..6758e781b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,10 @@ pilosa *.dot .idea/ .*.swp +.terraform/ +*.tfstate +launch.json +.terraform.lock.hcl +__pycache__/ +report.xml +outputs.json diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 0b7cc21e9..c32893e8c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -11,27 +11,22 @@ include: paths: - .go/pkg/mod/ variables: - GOVERSION: "1.16.9" + GOVERSION: "1.16.10" stages: - lint - test - build - integration - - #before_script: - #- echo "before_script" - #- git version - #- go env -w GOPRIVATE=github.com/molecula - #- mkdir -p .go - #- go version - #- go env -w GO111MODULE=on + - gauntlet golangci-lint: image: golangci/golangci-lint:v1.39.0 stage: lint extends: .go-cache allow_failure: false + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Checking for issues in new code" - golangci-lint run -v @@ -41,6 +36,8 @@ build lattice: image: node:14 variables: CI: "false" + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - cd lattice - yarn install @@ -59,6 +56,8 @@ run jest tests: image: node:14 variables: CI: "true" + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Testing lattice..." - cd lattice @@ -70,8 +69,10 @@ run jest tests: run go tests: stage: test - image: golang:1.16.10 + image: golang:$GOVERSION extends: .go-cache + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) @@ -84,6 +85,8 @@ run go tests future: stage: test image: golang:1.17.3 extends: .go-cache + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase unit tests..." - PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -) @@ -92,10 +95,11 @@ run go tests future: paths: - coverage.out - run go tests with output: stage: test - image: golang:1.16.10 + image: golang:$GOVERSION + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Running featurebase unit tests to capture JSON output..." - go test -json > test-report.out @@ -108,6 +112,8 @@ upload to sonarcloud: image: sonarsource/sonar-scanner-cli:4.6 variables: SONAR_TOKEN: $SONAR_TOKEN + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info needs: @@ -117,7 +123,9 @@ upload to sonarcloud: build for linux amd64: stage: build - image: golang:1.16.10 + image: golang:$GOVERSION + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - rm -r lattice - tar -xvf lattice.tar.gz @@ -130,7 +138,9 @@ build for linux amd64: build for linux arm64: stage: build - image: golang:1.16.10 + image: golang:$GOVERSION + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - rm -r lattice - tar -xvf lattice.tar.gz @@ -143,7 +153,9 @@ build for linux arm64: build for darwin amd64: stage: build - image: golang:1.16.10 + image: golang:$GOVERSION + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - rm -r lattice - tar -xvf lattice.tar.gz @@ -156,7 +168,9 @@ build for darwin amd64: build for darwin arm64: stage: build - image: golang:1.16.10 + image: golang:$GOVERSION + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - rm -r lattice - tar -xvf lattice.tar.gz @@ -169,7 +183,9 @@ build for darwin arm64: package for linux amd64: stage: build - image: golang:1.16.10 + image: golang:$GOVERSION + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' variables: GOOS: "linux" GOARCH: "amd64" @@ -190,6 +206,8 @@ build container fb: - "build for linux amd64" tags: - shell + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' before_script: - echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY} script: @@ -198,28 +216,102 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -# deploy EC2 instance, configure and run featurebase -deploy node for linux amd64: +smoke test: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: - PROFILE: "default" - AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY + PROFILE: "service-terraform" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + TF_VAR_cluster_prefix: "" + TF_VAR_branch: "" + rules: + - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - - chmod 400 gitlab-featurebase-dev.pem + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - - eval `ssh-agent -s` - - mkdir -p ~/.ssh - - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq + - apt update && apt -y install jq wget + - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - TF_VAR_branch=$CI_COMMIT_BRANCH + - echo "Branch --> $TF_VAR_branch" script: - - ./qa/scripts/deployNode.sh $PROFILE + - ./qa/scripts/setupSmokeTest.sh + - ./qa/scripts/testSmokeTest.sh + after_script: + - ./qa/scripts/teardownSmokeTest.sh needs: - - job: build for linux amd64 + - job: build for linux arm64 + artifacts: + when: always + paths: + - report.xml + reports: + junit: report.xml + +gauntlet: + stage: gauntlet + timeout: 4h + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "default" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + TF_VAR_cluster_prefix: "" + TF_VAR_branch: "" + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - TF_VAR_branch=$CI_COMMIT_BRANCH + - echo "Branch --> $TF_VAR_branch" + script: + - ./qa/scripts/setupSamsungGauntlet.sh + - ./qa/scripts/testSamsungGauntlet.sh + after_script: + - ./qa/scripts/teardownSamsungGauntlet.sh \ No newline at end of file diff --git a/Makefile b/Makefile index a24d8c40a..e33c92b56 100644 --- a/Makefile +++ b/Makefile @@ -149,12 +149,10 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml clustertests: vendor docker-compose -f $(DOCKER_COMPOSE) down docker-compose -f $(DOCKER_COMPOSE) build - docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 + docker-compose -f $(DOCKER_COMPOSE) up -d pilosa1 pilosa2 pilosa3 + docker-compose -f $(DOCKER_COMPOSE) run client1 + docker-compose -f $(DOCKER_COMPOSE) down -# Like clustertests, but rebuilds all images. -clustertests-build: vendor - docker-compose -f $(DOCKER_COMPOSE) down -v - docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build # Install Pilosa install: diff --git a/api.go b/api.go index 47558678d..fdee2f8f7 100644 --- a/api.go +++ b/api.go @@ -23,6 +23,7 @@ import ( "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/ingest" + "github.com/molecula/featurebase/v2/rbf" //"github.com/molecula/featurebase/v2/pg" "github.com/molecula/featurebase/v2/pql" @@ -130,9 +131,16 @@ func (api *API) SetAPIOptions(opts ...apiOption) error { var validAPIMethods = map[disco.ClusterState]map[apiMethod]struct{}{ disco.ClusterStateStarting: methodsCommon, disco.ClusterStateNormal: appendMap(methodsCommon, methodsNormal), - disco.ClusterStateDegraded: appendMap(methodsCommon, methodsDegraded), + // Ideally, this would be just `appendMap(methodsCommon, methodsDegraded)`, + // but in an attempt to reduce the influence that state (determined by etcd) + // has on a node under load, this is set to effectively allow all requests + // in a DEGRADED state. + disco.ClusterStateDegraded: appendMap(methodsCommon, methodsNormal), disco.ClusterStateResizing: appendMap(methodsCommon, methodsResizing), - disco.ClusterStateDown: methodsCommon, + // Ideally, this would be just `methodsCommon`, but in an attempt to reduce + // the influence that state (determined by etcd) has on a node under load, + // this is set to effectively allow all requests in a DOWN state. + disco.ClusterStateDown: appendMap(methodsCommon, methodsNormal), } func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} { @@ -3156,6 +3164,21 @@ func (api *API) Plan(ctx context.Context, q string) (*Stmt, error) { return api.server.PlanSQL(ctx, q) } +func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo { + infos := make(map[string]*rbf.DebugInfo) + + for key, dbShard := range api.holder.Txf().dbPerShard.Flatmap { + wrapper, ok := dbShard.W.(*RbfDBWrapper) + if !ok { + continue + } + + skey := fmt.Sprintf("%s/%d", key.index, key.shard) + infos[skey] = wrapper.db.DebugInfo() + } + return infos +} + type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` ReplicaN int `json:"replicaN"` diff --git a/api_test.go b/api_test.go index 9ffdea841..d42aca667 100644 --- a/api_test.go +++ b/api_test.go @@ -1415,3 +1415,25 @@ func TestVariousApiTranslateCalls(t *testing.T) { */ } } + +func TestAPI_RBFDebugInfo(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := test.MustRunCluster(t, 1, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node0"), + pilosa.OptServerClusterHasher(&offsetModHasher{}), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + )}, + ) + defer c.Close() + + coord := c.GetPrimary() + + if _, err := coord.API.CreateIndex(ctx, "i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if infos := coord.API.RBFDebugInfo(); infos == nil { + t.Fatal("expected info") + } +} diff --git a/authn/authenticate.go b/authn/authenticate.go index 6a5221600..a5567842b 100644 --- a/authn/authenticate.go +++ b/authn/authenticate.go @@ -1,4 +1,6 @@ // Copyright 2021 Molecula Corp. All rights reserved. + +// Package authn handles authentication package authn import ( @@ -31,8 +33,8 @@ type Auth struct { oAuthConfig *oauth2.Config } -// NewAuth is a constructor that returns a new auth object -func NewAuth(logger logger.Logger, url string, scopes []string, authUrl, tokenUrl, groupEndpoint, logout, clientID, clientSecret, hashKey, blockKey string) (*Auth, error) { +// NewAuth instantiates and returns a new Auth struct +func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, hashKey, blockKey string) (*Auth, error) { auth := &Auth{ logger: logger, cookieName: "molecula-chip", @@ -46,8 +48,8 @@ func NewAuth(logger logger.Logger, url string, scopes []string, authUrl, tokenUr ClientSecret: clientSecret, Scopes: scopes, Endpoint: oauth2.Endpoint{ - AuthURL: authUrl, - TokenURL: tokenUrl, + AuthURL: authURL, + TokenURL: tokenURL, }, }, } @@ -65,30 +67,37 @@ func NewAuth(logger logger.Logger, url string, scopes []string, authUrl, tokenUr return auth, nil } -type CookieValue struct { +// AuthContext holds the value of an authenticated user's cookie +type AuthContext struct { UserID string UserName string GroupMembership []Group Token *oauth2.Token } -type Groups struct { - Groups []Group `json:"value"` -} - +// Group holds group information for an authenticated user type Group struct { UserID string GroupID string `json:"id"` GroupName string `json:"displayName"` } +// Groups holds a slice of Group informations for marshalling from Json +type Groups struct { + Groups []Group `json:"value"` +} + +// UserInfo holds user information for an authenticated user type UserInfo struct { UserID string `json:"userid"` UserName string `json:"username"` } -// Authenticate reads and validates a cookie, redirects if invalid or missing, otherwise returns -// the group membership information stored in the cookie. +// Authenticate reads the authentication cookie from a request, returning the +// user's group memberships on success. If the cookie is not present or has expired, +// Authenticate redirects the user to sign in. If the cookie is within the +// refresh window of expiring, the cookie is refreshed, and the updated group +// membership is returned. func (a *Auth) Authenticate(w http.ResponseWriter, r *http.Request) ([]Group, error) { cookie, err := a.readCookie(w, r) if err != nil { @@ -112,10 +121,10 @@ func (a *Auth) Authenticate(w http.ResponseWriter, r *http.Request) ([]Group, er } -// Login redirects user to the IdP authorize endpoint for auth code +// Login redirects a user to login to their configured oAuth authorize endpoint func (a *Auth) Login(w http.ResponseWriter, r *http.Request) { - authUrl := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL) - http.Redirect(w, r, authUrl, http.StatusTemporaryRedirect) + authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL) + http.Redirect(w, r, authURL, http.StatusTemporaryRedirect) } // Logout clears out user cookie and redirects user to IdP's logout endpoint @@ -125,7 +134,8 @@ func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, redirect, http.StatusTemporaryRedirect) } -// Redirect gets user information from IdP and sets a secure cookie +// Redirect handles the oAuth /redirect endpoint. It gets user information from +// the identity provider and sets a secure cookie holding the user information. func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) { code := r.FormValue("code") token, err := a.getToken(r, code) @@ -135,7 +145,7 @@ func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) { return } - cv, err := a.newCookieValue(token) + cv, err := a.newAuthContext(token) if err != nil || cv == nil { a.logger.Warnf("creating cookie: %+v", err) http.Error(w, "Bad Request: 400", http.StatusBadRequest) @@ -154,11 +164,11 @@ func (a *Auth) GetUserInfo(w http.ResponseWriter, r *http.Request) *UserInfo { a.logger.Warnf("was not able to read cookie for req: %+v", r) return &resp } + return &UserInfo{ UserID: cookie.UserID, UserName: cookie.UserName, } - } // getToken exhanges authorization code for an oAuth2 token @@ -170,8 +180,8 @@ func (a *Auth) getToken(r *http.Request, code string) (*oauth2.Token, error) { return token, nil } -// newCookieValue parses a jwt `token` and returns relevant information in a cookie value struct -func (a *Auth) newCookieValue(token *oauth2.Token) (*CookieValue, error) { +// newAuthContext parses a jwt `token` and returns relevant information in a cookie value struct +func (a *Auth) newAuthContext(token *oauth2.Token) (*AuthContext, error) { if token == nil { return nil, errors.New("baking cookie due to nil token") } @@ -195,7 +205,7 @@ func (a *Auth) newCookieValue(token *oauth2.Token) (*CookieValue, error) { } // not needed at this point in the logic and makes the encoded cookie too large token.AccessToken = "" - return &CookieValue{ + return &AuthContext{ UserID: claims["oid"].(string), UserName: claims["name"].(string), GroupMembership: groups.Groups, @@ -206,6 +216,7 @@ func (a *Auth) newCookieValue(token *oauth2.Token) (*CookieValue, error) { // getGroupMembership uses a oauth2 token to retrieve group membership information from IdP func (a *Auth) getGroupMembership(token *oauth2.Token) (Groups, error) { var groups Groups + req, err := http.NewRequest("GET", a.groupEndpoint, nil) if err != nil { return groups, errors.Wrap(err, "creating new request to group endpoint") @@ -231,13 +242,13 @@ func (a *Auth) getGroupMembership(token *oauth2.Token) (Groups, error) { } // readCookie decodes an encrypted and signed cookie and returns the contained info -func (a *Auth) readCookie(w http.ResponseWriter, r *http.Request) (*CookieValue, error) { +func (a *Auth) readCookie(w http.ResponseWriter, r *http.Request) (*AuthContext, error) { cookie, err := r.Cookie(a.cookieName) if err != nil { return nil, errors.Wrap(err, "cookie not found") } - var value CookieValue + var value AuthContext err = a.secure.Decode(a.cookieName, cookie.Value, &value) if err != nil { http.SetCookie(w, a.getEmptyCookie()) @@ -247,10 +258,10 @@ func (a *Auth) readCookie(w http.ResponseWriter, r *http.Request) (*CookieValue, return &value, nil } -func (a *Auth) setCookie(w http.ResponseWriter, cookie *CookieValue) error { +func (a *Auth) setCookie(w http.ResponseWriter, cookie *AuthContext) error { encoded, err := a.secure.Encode(a.cookieName, cookie) if err != nil { - return errors.Wrap(err, "encoding CookieValue") + return errors.Wrap(err, "encoding AuthContext") } http.SetCookie(w, &http.Cookie{ @@ -265,9 +276,9 @@ func (a *Auth) setCookie(w http.ResponseWriter, cookie *CookieValue) error { return nil } -func (a *Auth) refreshToken(w http.ResponseWriter, cookie *CookieValue) error { +func (a *Auth) refreshToken(w http.ResponseWriter, cookie *AuthContext) error { if cookie.Token.RefreshToken == "" { - return errors.New("no refresh token found, check auth scopes to see if refresh tokens are being provided by your IdP.") + return errors.New("no refresh token found, check auth scopes to see if refresh tokens are being provided by your IdP") } tokenSource := a.oAuthConfig.TokenSource(context.Background(), cookie.Token) newToken, err := tokenSource.Token() @@ -276,7 +287,7 @@ func (a *Auth) refreshToken(w http.ResponseWriter, cookie *CookieValue) error { } if newToken.Expiry != cookie.Token.Expiry { - cv, err := a.newCookieValue(newToken) + cv, err := a.newAuthContext(newToken) if err != nil { return errors.Wrap(err, "creating cookie value from token") } @@ -307,4 +318,5 @@ func (a *Auth) getEmptyCookie() *http.Cookie { HttpOnly: true, SameSite: http.SameSiteStrictMode, } + } diff --git a/authn/authenticate_internal_test.go b/authn/authenticate_internal_test.go index 860283433..7af62092e 100644 --- a/authn/authenticate_internal_test.go +++ b/authn/authenticate_internal_test.go @@ -13,7 +13,7 @@ import ( func TestAuth(t *testing.T) { var ( - ClientId = "e9088663-eb08-41d7-8f65-efb5f54bbb71" + ClientID = "e9088663-eb08-41d7-8f65-efb5f54bbb71" ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" AuthorizeURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize" TokenURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token" @@ -32,7 +32,7 @@ func TestAuth(t *testing.T) { TokenURL, GroupEndpointURL, LogoutURL, - ClientId, + ClientID, ClientSecret, Key, Key, @@ -56,7 +56,7 @@ func TestAuth(t *testing.T) { GroupID: "abcd123-A", GroupName: "Romantic Painters", } - validCV := CookieValue{ + validCV := AuthContext{ UserID: "snowstorm", UserName: "J.M.W. Turner", GroupMembership: []Group{grp}, @@ -94,7 +94,7 @@ func TestAuth(t *testing.T) { TokenURL, GroupEndpointURL, LogoutURL, - ClientId, + ClientID, ClientSecret, Key, ShortKey, @@ -103,15 +103,15 @@ func TestAuth(t *testing.T) { t.Fatalf("expected error decoding block key got: %v", err) } }) - t.Run("NewCookieValue-BadAccessToken", func(t *testing.T) { - _, err := a.newCookieValue(&tokenAT) + t.Run("NewAuthContext-BadAccessToken", func(t *testing.T) { + _, err := a.newAuthContext(&tokenAT) if err == nil || !strings.Contains(err.Error(), "jwt claims") { t.Fatalf("expected failure regarding jwt claims, got: %v", err) } }) - t.Run("CookieValue-NoAccessToken", func(t *testing.T) { - _, err := a.newCookieValue(&tokenNoAT) + t.Run("AuthContext-NoAccessToken", func(t *testing.T) { + _, err := a.newAuthContext(&tokenNoAT) if err == nil || !strings.Contains(err.Error(), "access token") { t.Fatalf("expected failure regarding access token, got: %v", err) } diff --git a/client.go b/client.go index fe91eb124..75741fcd3 100644 --- a/client.go +++ b/client.go @@ -80,8 +80,14 @@ type InternalClient interface { GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) - ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error - ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error + // ImportFieldKeys and ImportIndexKeys are mainly used when + // restoring a backup. They take a readerFunc which returns a + // reader rather than taking an io.Reader directly to allow for + // efficient retries (rather than reading the entire request body + // into a buffer and reusing it). Reader returned from the func + // must be properly closed by the implementation. + ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error + ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error // SetInternalAPI tells the client the API it should use for internal/loopback ops // where applicable. @@ -277,11 +283,11 @@ func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) { return nil, nil } -func (c nopInternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error { +func (c nopInternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error { return nil } -func (c nopInternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error { +func (c nopInternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error { return nil } diff --git a/cmd/backup.go b/cmd/backup.go index 5761b6d57..28712123d 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -23,11 +23,13 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file. } flags := ccmd.Flags() - flags.StringVarP(&cmd.OutputDir, "output", "o", "", "output dir to write to") - flags.BoolVar(&cmd.NoSync, "no-sync", false, "disable file sync") - flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "number of concurrent backup goroutines") - flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.") - flags.StringVar(&cmd.Index, "index", "", "index to backup, default backs up all indexes. ") + flags.StringVarP(&cmd.OutputDir, "output", "o", "", "Output directory to write to.") + flags.BoolVar(&cmd.NoSync, "no-sync", false, "Disable file sync") + flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "Number of concurrent backup goroutines.") + flags.StringVar(&cmd.Host, "host", "localhost:10101", "The address (host:port) of FeatureBase (HTTP).") + flags.StringVar(&cmd.Index, "index", "", "Index to backup, default backs up all indexes. ") + flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.") + flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.") ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification) return ccmd } diff --git a/cmd/restore.go b/cmd/restore.go index e9af62d24..bc9271d0e 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -25,6 +25,8 @@ The Restore command will take a backup archive and restore it to a new, clean cl flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream") flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.") flags.IntVar(&cmd.Concurrency, "concurrency", 1, "number of concurrent uploads") + flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.") + flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.") ctl.SetTLSConfig( flags, "", &cmd.TLS.CertificatePath, diff --git a/cmd/slurp/slurp.go b/cmd/slurp/slurp.go index 446db53b1..800a5a773 100644 --- a/cmd/slurp/slurp.go +++ b/cmd/slurp/slurp.go @@ -92,8 +92,10 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { byteData, err := ioutil.ReadAll(tr) vprint.PanicOn(err) - br := bytes.NewReader(byteData) - err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, br) + readerFunc := func() (io.Reader, error) { + return bytes.NewReader(byteData), nil + } + err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, readerFunc) if err != nil { return err } @@ -106,9 +108,11 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error { } byteData, err := ioutil.ReadAll(tr) vprint.PanicOn(err) + readerFunc := func() (io.Reader, error) { + return bytes.NewReader(byteData), nil + } - br := bytes.NewReader(byteData) - err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, br) + err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, readerFunc) if err != nil { return err } diff --git a/ctl/backup.go b/ctl/backup.go index 0e8a257f1..302041dfe 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -10,11 +10,13 @@ import ( "io/ioutil" "os" "path/filepath" + "time" pilosa "github.com/molecula/featurebase/v2" - "github.com/molecula/featurebase/v2/http" + fb_http "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/server" "github.com/molecula/featurebase/v2/topology" + "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -37,6 +39,12 @@ type BackupCommand struct { // nolint: maligned // Number of concurrent backup goroutines running at a time. Concurrency int + // Amount of time after first failed request to continue retrying. + RetryPeriod time.Duration `json:"retry-period"` + + // Host:port on which to listen for pprof. + Pprof string `json:"pprof"` + // Reusable client. client pilosa.InternalClient @@ -51,11 +59,20 @@ func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand return &BackupCommand{ CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), Concurrency: 1, + RetryPeriod: time.Minute, + Pprof: "localhost:43809", } } // Run executes the main program execution. func (cmd *BackupCommand) Run(ctx context.Context) (err error) { + logger := cmd.Logger() + close, err := startProfilingServer(cmd.Pprof, logger) + if err != nil { + return errors.Wrap(err, "starting profiling server") + } + defer close() + // Validate arguments. if cmd.OutputDir == "" { return fmt.Errorf("-o flag required") @@ -70,7 +87,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) { } // Create a client to the server. - client, err := commandClient(cmd) + client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod)) if err != nil { return fmt.Errorf("creating client: %w", err) } @@ -262,7 +279,7 @@ func (cmd *BackupCommand) backupShardNode(ctx context.Context, indexName string, logger := cmd.Logger() logger.Printf("backing up shard: index=%q id=%d", indexName, shard) - client := http.NewInternalClientFromURI(&node.URI, http.GetHTTPClient(cmd.tlsConfig)) + client := fb_http.NewInternalClientFromURI(&node.URI, fb_http.GetHTTPClient(cmd.tlsConfig), fb_http.WithClientRetryPeriod(cmd.RetryPeriod)) rc, err := client.ShardReader(ctx, indexName, shard) if err != nil { return fmt.Errorf("fetching shard reader: %w", err) diff --git a/ctl/common.go b/ctl/common.go index c23b42f4c..558cdece3 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -2,6 +2,11 @@ package ctl import ( + "net" + "time" + + gohttp "net/http" + "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/server" @@ -25,14 +30,22 @@ func SetTLSConfig(flags *pflag.FlagSet, prefix string, certificatePath *string, flags.BoolVarP(enableClientVerification, prefix+"tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections") } +// default dial timeout is 30s for some reason which makes testing +// failures/retries really awkward. I don't think we need it that +// high, so I set it to 1s here... let's see what happens. +func clientOptions(client *gohttp.Client, dialer *net.Dialer) *gohttp.Client { + dialer.Timeout = time.Second * 1 + return client +} + // commandClient returns a pilosa.InternalHTTPClient for the command -func commandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) { +func commandClient(cmd CommandWithTLSSupport, opts ...http.InternalClientOption) (*http.InternalClient, error) { tls := cmd.TLSConfiguration() tlsConfig, err := server.GetTLSConfig(&tls, cmd.Logger()) if err != nil { return nil, errors.Wrap(err, "getting tls config") } - client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(tlsConfig)) + client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(tlsConfig, clientOptions), opts...) if err != nil { return nil, errors.Wrap(err, "getting internal client") } diff --git a/ctl/rbf_check.go b/ctl/rbf_check.go index 00594b861..f3d40912d 100644 --- a/ctl/rbf_check.go +++ b/ctl/rbf_check.go @@ -26,7 +26,7 @@ func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckComm } } -// Run executes the export. +// Run executes a consistency check of an RBF database. func (cmd *RBFCheckCommand) Run(ctx context.Context) error { // Open database. db := rbf.NewDB(cmd.Path, nil) @@ -37,7 +37,15 @@ func (cmd *RBFCheckCommand) Run(ctx context.Context) error { // Run check on the database. if err := db.Check(); err != nil { - return err + switch err := err.(type) { + case rbf.ErrorList: + for i := range err { + fmt.Fprintln(cmd.Stdout, err[i]) + } + default: + fmt.Fprintln(cmd.Stdout, err) + } + return fmt.Errorf("check failed") } // If successful, print a success message. diff --git a/ctl/rbf_check_test.go b/ctl/rbf_check_test.go new file mode 100644 index 000000000..c11cd30dc --- /dev/null +++ b/ctl/rbf_check_test.go @@ -0,0 +1,33 @@ +// Copyright 2021 Molecula Corp. All rights reserved. +package ctl + +import ( + "bytes" + "context" + "path/filepath" + "testing" +) + +func TestRBFCheckCommand_Run(t *testing.T) { + t.Run("OK", func(t *testing.T) { + var stdout, stderr bytes.Buffer + cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-check", "ok") + if err := cmd.Run(context.Background()); err != nil { + t.Fatal(err) + } else if got, want := stdout.String(), `ok`+"\n"; got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) + + t.Run("ErrInvalidPageType", func(t *testing.T) { + var stdout, stderr bytes.Buffer + cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-check", "err-invalid-page-type") + if err := cmd.Run(context.Background()); err == nil || err.Error() != `check failed` { + t.Fatal(err) + } else if got, want := stdout.String(), `page not in-use & not free: pgno=4`+"\n"; got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) +} diff --git a/ctl/rbf_pages.go b/ctl/rbf_pages.go index ca3484f31..b774175ae 100644 --- a/ctl/rbf_pages.go +++ b/ctl/rbf_pages.go @@ -49,7 +49,16 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error { // Iterate over each page and grab info. infos, err := tx.PageInfos() if err != nil { - return err + fmt.Fprintln(cmd.Stdout, "ERRORS:") + switch err := err.(type) { + case rbf.ErrorList: + for i := range err { + fmt.Fprintln(cmd.Stdout, err[i]) + } + default: + fmt.Fprintln(cmd.Stdout, err) + } + fmt.Fprintln(cmd.Stdout, "") } // Write header. diff --git a/ctl/rbf_pages_test.go b/ctl/rbf_pages_test.go new file mode 100644 index 000000000..73c395e1b --- /dev/null +++ b/ctl/rbf_pages_test.go @@ -0,0 +1,52 @@ +// Copyright 2021 Molecula Corp. All rights reserved. +package ctl + +import ( + "bytes" + "context" + "path/filepath" + "testing" +) + +func TestRBFPagesCommand_Run(t *testing.T) { + t.Run("OK", func(t *testing.T) { + want := ` +ID TYPE EXTRA +======== ========== ==================== +0 meta pageN=4,walid=4,rootrec=1,freelist=2 +1 rootrec next=0 +2 leaf flags=x2,celln=0 +3 leaf flags=x2,celln=1 +`[1:] + + var stdout, stderr bytes.Buffer + cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-pages", "ok") + if err := cmd.Run(context.Background()); err != nil { + t.Fatal(err) + } else if got := stdout.String(); got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) + + t.Run("ErrInvalidPageType", func(t *testing.T) { + want := ` +ID TYPE EXTRA +======== ========== ==================== +0 meta pageN=5,walid=4,rootrec=1,freelist=2 +1 rootrec next=0 +2 leaf flags=x2,celln=0 +3 leaf flags=x2,celln=1 +4 unknown [] +`[1:] + + var stdout, stderr bytes.Buffer + cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr) + cmd.Path = filepath.Join("testdata", "rbf-pages", "err-invalid-page-type") + if err := cmd.Run(context.Background()); err != nil { + t.Fatal(err) + } else if got := stdout.String(); got != want { + t.Fatalf("got:\n%s\n\nwant:\n%s", got, want) + } + }) +} diff --git a/ctl/restore.go b/ctl/restore.go index 373581d5d..b7f1fb2dc 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -5,18 +5,23 @@ import ( "context" "crypto/tls" "encoding/json" - "errors" "fmt" "io" + "math" "net/http" "os" "path/filepath" "strconv" "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" pilosa "github.com/molecula/featurebase/v2" + fb_http "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/server" "github.com/molecula/featurebase/v2/topology" + "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -29,6 +34,13 @@ type RestoreCommand struct { // Filepath to the backup file. Path string + + // Amount of time after first failed request to continue retrying. + RetryPeriod time.Duration `json:"retry-period"` + + // Host:port on which to listen for pprof. + Pprof string `json:"pprof"` + // Reusable client. client pilosa.InternalClient @@ -41,13 +53,20 @@ type RestoreCommand struct { func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand { return &RestoreCommand{ CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), + RetryPeriod: time.Second * 30, Concurrency: 1, + Pprof: "localhost:43809", } } // Run executes the restore. func (cmd *RestoreCommand) Run(ctx context.Context) (err error) { logger := cmd.Logger() + close, err := startProfilingServer(cmd.Pprof, logger) + if err != nil { + return errors.Wrap(err, "starting profiling server") + } + defer close() // Validate arguments. if cmd.Path == "" { @@ -62,7 +81,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) { return fmt.Errorf("parsing tls config: %w", err) } // Create a client to the server. - client, err := commandClient(cmd) + client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod)) if err != nil { return fmt.Errorf("creating client: %w", err) } @@ -119,7 +138,7 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology. if len(existingSchema) == 0 { cmd.Logger().Printf("Load Schema") url := primary.URI.Path("/schema") - var client http.Client + client := cmd.newClient() _, err = client.Post(url, "application/json", f) } else { schema := &pilosa.Schema{} @@ -159,6 +178,34 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology. return err } +func retryWith400(ctx context.Context, resp *http.Response, err error) (bool, error) { + if resp != nil && resp.StatusCode >= 400 { // we have some dumb status codes + return true, nil + } + return retryablehttp.DefaultRetryPolicy(ctx, resp, err) +} + +// This logic is taken from featurebase/http/client.go If this logic +// is not the same as what's there, that could be a problem. Ideally +// all network calls from restore would go through the client and this +// would not longer be needed. +func (cmd *RestoreCommand) newClient() *retryablehttp.Client { + min := time.Millisecond * 100 + + // do some math to figure out how many attempts we need to get our + // total sleep time close to the period + attempts := math.Log2(float64(cmd.RetryPeriod)) - math.Log2(float64(min)) + attempts += 0.3 // mmmm, fudge + if attempts < 1 { + attempts = 1 + } + client := retryablehttp.NewClient() + client.RetryWaitMin = min + client.RetryMax = int(attempts) + client.CheckRetry = retryWith400 + return client +} + func (cmd *RestoreCommand) restoreIDAlloc(ctx context.Context, primary *topology.Node) error { logger := cmd.Logger() @@ -174,7 +221,7 @@ func (cmd *RestoreCommand) restoreIDAlloc(ctx context.Context, primary *topology logger.Printf("Load idalloc") url := primary.URI.Path("/internal/idalloc/restore") - var client http.Client + client := cmd.newClient() _, err = client.Post(url, "application/octet-stream", f) return err } @@ -244,14 +291,14 @@ func (cmd *RestoreCommand) restoreShard(ctx context.Context, filename string) er defer f.Close() url := node.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard)) - req, err := http.NewRequest("POST", url, f) + req, err := retryablehttp.NewRequest("POST", url, f) if err != nil { return err } req = req.WithContext(ctx) req.Header.Set("Content-Type", "application/octet-stream") - var client http.Client + client := cmd.newClient() resp, err := client.Do(req) if err != nil { return err @@ -319,13 +366,11 @@ func (cmd *RestoreCommand) restoreIndexTranslationFile(ctx context.Context, file for _, node := range nodes { if err := func() error { - f, err := os.Open(filename) - if err != nil { - return err + readerFunc := func() (io.Reader, error) { + return os.Open(filename) // gets used as an HTTP request body and closed by http library } - defer f.Close() - return cmd.client.ImportIndexKeys(ctx, &node.URI, indexName, partitionID, false, f) + return cmd.client.ImportIndexKeys(ctx, &node.URI, indexName, partitionID, false, readerFunc) }(); err != nil { return err } @@ -380,13 +425,11 @@ func (cmd *RestoreCommand) restoreFieldTranslationFile(ctx context.Context, node for _, node := range nodes { if err := func() error { - f, err := os.Open(filename) - if err != nil { - return err + readerFunc := func() (io.Reader, error) { + return os.Open(filename) } - defer f.Close() - return cmd.client.ImportFieldKeys(ctx, &node.URI, indexName, fieldName, false, f) + return cmd.client.ImportFieldKeys(ctx, &node.URI, indexName, fieldName, false, readerFunc) }(); err != nil { return err } diff --git a/ctl/server.go b/ctl/server.go index 2f1f0ee65..77b42d4cb 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -45,7 +45,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Etcd // Etcd.Name used Config.Name for its value. - // Etcd.Dir defaults to a directory under the pilosa data directory. + flags.StringVar(&srv.Config.Etcd.Dir, "etcd.dir", srv.Config.Etcd.Dir, "Directory to store etcd data files. If not provided, a directory will be created under the main data-dir directory.") // Etcd.ClusterName uses Cluster.Name for its value flags.StringVar(&srv.Config.Etcd.LClientURL, "etcd.listen-client-address", srv.Config.Etcd.LClientURL, "Listen client address.") flags.StringVar(&srv.Config.Etcd.AClientURL, "etcd.advertise-client-address", srv.Config.Etcd.AClientURL, "Advertise client address. If not provided, uses the listen client address.") diff --git a/ctl/testdata/rbf-check/err-invalid-page-type/data b/ctl/testdata/rbf-check/err-invalid-page-type/data new file mode 100644 index 000000000..f088c25c9 Binary files /dev/null and b/ctl/testdata/rbf-check/err-invalid-page-type/data differ diff --git a/ctl/testdata/rbf-check/err-invalid-page-type/wal b/ctl/testdata/rbf-check/err-invalid-page-type/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/testdata/rbf-check/ok/data b/ctl/testdata/rbf-check/ok/data new file mode 100644 index 000000000..e4c3b621e Binary files /dev/null and b/ctl/testdata/rbf-check/ok/data differ diff --git a/ctl/testdata/rbf-check/ok/wal b/ctl/testdata/rbf-check/ok/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/testdata/rbf-pages/err-invalid-page-type/data b/ctl/testdata/rbf-pages/err-invalid-page-type/data new file mode 100644 index 000000000..f088c25c9 Binary files /dev/null and b/ctl/testdata/rbf-pages/err-invalid-page-type/data differ diff --git a/ctl/testdata/rbf-pages/err-invalid-page-type/wal b/ctl/testdata/rbf-pages/err-invalid-page-type/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/testdata/rbf-pages/ok/data b/ctl/testdata/rbf-pages/ok/data new file mode 100644 index 000000000..e4c3b621e Binary files /dev/null and b/ctl/testdata/rbf-pages/ok/data differ diff --git a/ctl/testdata/rbf-pages/ok/wal b/ctl/testdata/rbf-pages/ok/wal new file mode 100644 index 000000000..e69de29bb diff --git a/ctl/util.go b/ctl/util.go new file mode 100644 index 000000000..60ac082c9 --- /dev/null +++ b/ctl/util.go @@ -0,0 +1,56 @@ +package ctl + +import ( + "context" + "net" + "net/http" + "net/http/pprof" + "runtime" + "time" + + "github.com/felixge/fgprof" + "github.com/molecula/featurebase/v2/logger" + "github.com/pkg/errors" +) + +// startProfilingServer starts a server which handles /debug/pprof and +// /debug/fgprof for use in utilities we might want to profile but +// wouldn't otherwise be running an http server. Caller should call +// the returned close function before exiting to release resources. +func startProfilingServer(addr string, logger logger.Logger) (close func() error, err error) { + if addr == "" { + return func() error { return nil }, nil + } + + sm := http.NewServeMux() + sm.Handle("/debug/fgprof", fgprof.Handler()) + sm.HandleFunc("/debug/pprof/", pprof.Index) + sm.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + sm.HandleFunc("/debug/pprof/profile", pprof.Profile) + sm.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + sm.HandleFunc("/debug/pprof/trace", pprof.Trace) + s := &http.Server{ + Addr: addr, + Handler: sm, + } + runtime.SetBlockProfileRate(10000000) // 1 sample per 10 ms + runtime.SetMutexProfileFraction(100) // 1% sampling + ln, err := net.Listen("tcp", addr) + if err != nil { + return nil, err + } + go func() { + logger.Printf("Listening for /debug/pprof/ and /debug/fgprof on '%s'", addr) + logger.Printf("%v", s.Serve(ln)) + }() + + return func() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + err := s.Shutdown(ctx) + if err != nil { + return errors.Wrap(err, "shutting down profiling server") + } + return s.Close() + }, nil +} diff --git a/executor.go b/executor.go index 5e03e1a72..b9d83e441 100644 --- a/executor.go +++ b/executor.go @@ -179,11 +179,18 @@ func newExecutor(opts ...executorOption) *executor { func (e *executor) addWorker() { e.workersWG.Add(1) - atomic.AddInt64(&e.currentWorkers, 1) + n := atomic.AddInt64(&e.currentWorkers, 1) + if e.Holder != nil { + e.Holder.Stats.Gauge("worker_total", float64(n), 0) + } + go func() { defer e.workersWG.Done() e.worker(e.work) - atomic.AddInt64(&e.currentWorkers, -1) + n := atomic.AddInt64(&e.currentWorkers, -1) + if e.Holder != nil { + e.Holder.Stats.Gauge("worker_total", float64(n), 0) + } }() } @@ -204,6 +211,14 @@ func (e *executor) Close() error { return nil } +// InitStats initializes stats counters. Must be called after Holder set. +func (e *executor) InitStats() { + if e.Holder != nil { + e.Holder.Stats.Count("job_total", 0, 0) + e.Holder.Stats.Gauge("worker_total", float64(atomic.LoadInt64(&e.currentWorkers)), 0) + } +} + // Execute executes a PQL query. func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute") @@ -587,6 +602,11 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q return nil, err } + if vc, ok := v.(ValCount); ok { + vc.cleanup() + v = vc + } + results = append(results, v) // Some Calls can have significant data associated with them // that gets generated during processing, such as Precomputed @@ -597,6 +617,22 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q return results, nil } +// cleanup removes the integer value (Val) from the ValCount if one of +// the other fields is in use. +// +// ValCounts are normally holding data which is stored as a BSI +// (integer) under the hood. Sometimes it's convenient to be able to +// compare the underlying integer values rather than their +// interpretation as decimal, timestamp, etc, so the lower level +// functions may return both integer and the interpreted value, but we +// don't want to pass that all the way back to the client, so we +// remove it here. +func (vc *ValCount) cleanup() { + if vc.Val != 0 && (vc.FloatVal != 0 || !vc.TimestampVal.IsZero() || vc.DecimalVal != nil) { + vc.Val = 0 + } +} + // preprocessQuery expands any calls that need preprocessing. func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*pql.Call, error) { switch c.Name { @@ -1261,6 +1297,10 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string if err != nil { return ValCount{}, errors.New("Percentile(): field required") } + field := e.Holder.Field(index, fieldName) + if field == nil { + return ValCount{}, ErrFieldNotFound + } // filter call for min & max var filterCall *pql.Call @@ -1281,7 +1321,7 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string return ValCount{}, errors.Wrap(err, "executing Min call for Percentile") } if nthFloat == 0.0 { - return ValCount{Val: minVal.Val, Count: minVal.Count}, nil + return minVal, nil } // get max @@ -1348,11 +1388,11 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string } else if leftCountWeighted < rightCount { min = possibleNthVal + 1 } else { - return ValCount{Val: possibleNthVal, Count: 1}, nil + return field.valCountize(possibleNthVal, 1, nil) } } - return ValCount{Val: min, Count: 1}, nil + return field.valCountize(min, 1, nil) } @@ -5989,6 +6029,7 @@ type job struct { func (e *executor) worker(work chan job) { for j := range work { atomic.AddUint64(&e.workCounter, 1) + e.Holder.Stats.Count("job_total", 1, 0) if j.idleHands { return } @@ -8130,6 +8171,8 @@ func getScaledInt(f *Field, v interface{}) (int64, error) { switch tv := v.(type) { case time.Time: value = tv.UnixNano() / TimeUnitNanos(f.options.TimeUnit) + case int64: + value = tv default: return 0, errors.Errorf("unexpected timestamp value type %T, val %v", tv, tv) } diff --git a/executor_internal_test.go b/executor_internal_test.go index 5c5ed9314..79a9cfe72 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -489,3 +489,18 @@ func TestExecutorSafeCopyDistinctTimestamp(t *testing.T) { t.Fatalf("Did not copy results. got %+v, want %+v", copied.Results, response.Results) } } + +func TestGetScaledInt(t *testing.T) { + f := OpenField(t, OptFieldTypeTimestamp(time.Now(), "ms")) + defer f.Close() + // check that fields with type timestamp return the int64 passed in to getScaledInt with nil err + v := time.Now().Unix() + res, err := getScaledInt(f.Field, v) + if err != nil { + t.Errorf("got error %v, expected nil", err) + } + if !reflect.DeepEqual(res, v) { + t.Errorf("expected %v, got %v", v, res) + } + +} diff --git a/field.go b/field.go index d325c477e..e45e82d6e 100644 --- a/field.go +++ b/field.go @@ -1388,18 +1388,7 @@ func (f *Field) MaxForShard(tx Tx, shard uint64, filter *Row) (ValCount, error) return ValCount{}, errors.Wrap(err, "calling fragment.max") } - valCount := ValCount{Count: int64(cnt)} - - if f.Options().Type == FieldTypeDecimal { - dec := pql.NewDecimal(max+bsig.Base, bsig.Scale) - valCount.DecimalVal = &dec - } else if f.Options().Type == FieldTypeTimestamp { - valCount.TimestampVal = time.Unix(0, (max+bsig.Base)*TimeUnitNanos(f.options.TimeUnit)).UTC() - } else { - valCount.Val = max + bsig.Base - } - - return valCount, nil + return f.valCountize(max, cnt, bsig) } // MinForShard returns the minimum value which appears in this shard @@ -1434,17 +1423,32 @@ func (f *Field) MinForShard(tx Tx, shard uint64, filter *Row) (ValCount, error) return ValCount{}, errors.Wrap(err, "calling fragment.min") } + return f.valCountize(min, cnt, bsig) +} + +// valCountize takes the "raw" value and count we get from the +// fragment and calculates the cooked values for this field +// (timestamping, decimaling, or just adding in the base). It always +// includes the int64 "Val\" value to make comparisons easier in the +// executor (at time of writing, Percentile takes advantage of this, +// but we might be able to simplify logic in other places as well). +func (f *Field) valCountize(val int64, cnt uint64, bsig *bsiGroup) (ValCount, error) { + if bsig == nil { + bsig = f.bsiGroup(f.name) + if bsig == nil { + return ValCount{}, ErrBSIGroupNotFound + } + + } valCount := ValCount{Count: int64(cnt)} if f.Options().Type == FieldTypeDecimal { - dec := pql.NewDecimal(min+bsig.Base, bsig.Scale) + dec := pql.NewDecimal(val+bsig.Base, bsig.Scale) valCount.DecimalVal = &dec } else if f.Options().Type == FieldTypeTimestamp { - valCount.TimestampVal = time.Unix(0, (min+bsig.Base)*TimeUnitNanos(f.options.TimeUnit)).UTC() - } else { - valCount.Val = min + bsig.Base + valCount.TimestampVal = time.Unix(0, (val+bsig.Base)*TimeUnitNanos(f.options.TimeUnit)).UTC() } - + valCount.Val = val + bsig.Base return valCount, nil } diff --git a/field_internal_test.go b/field_internal_test.go index f1219f7e2..d9471db91 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -182,6 +182,23 @@ func TestBSIGroup_BaseValue(t *testing.T) { }) } +func TestField_ValCountize(t *testing.T) { + f := OpenField(t, OptFieldTypeDefault()) + defer f.Close() + // check that you get an empty val count and err + // BSIGroupNotFound on nil bsig from + // f.bsiGroup(f.name) + f.bsiGroups = []*bsiGroup{} + v, err := f.valCountize(42, 42, nil) + if !reflect.DeepEqual(v, ValCount{}) { + t.Errorf("expected %v, got %v", ValCount{}, v) + } + if err != ErrBSIGroupNotFound { + t.Errorf("expected %v, got %v", ErrBSIGroupNotFound, err) + } + +} + // Ensure field can open and retrieve a view. func TestField_DeleteView(t *testing.T) { f := OpenField(t, OptFieldTypeDefault()) @@ -748,29 +765,29 @@ func TestDecimalField_MinMaxForShard(t *testing.T) { name: "single", columnIDs: []uint64{1}, values: []float64{10.1}, - expMax: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1}, - expMin: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1}, + expMax: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1}, + expMin: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1}, }, { name: "twovals", columnIDs: []uint64{1, 2}, values: []float64{10.1, 20.2}, - expMax: ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1}, - expMin: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1}, + expMax: ValCount{Val: 20200, DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1}, + expMin: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1}, }, { name: "multiplecounts", columnIDs: []uint64{1, 2, 3, 4, 5}, values: []float64{10.1, 20.2, 10.1, 10.1, 20.2}, - expMax: ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2}, - expMin: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3}, + expMax: ValCount{Val: 20200, DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2}, + expMin: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3}, }, { name: "middlevals", columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, values: []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11}, - expMax: ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2}, - expMin: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3}, + expMax: ValCount{Val: 20200, DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2}, + expMin: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3}, }, } { t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { diff --git a/go.mod b/go.mod index 24e125060..3b36cd0a8 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 github.com/gorilla/securecookie v1.1.1 + github.com/hashicorp/go-retryablehttp v0.7.0 github.com/improbable-eng/grpc-web v0.13.0 github.com/lib/pq v1.8.0 github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b diff --git a/go.sum b/go.sum index ed75692df..97504ff52 100644 --- a/go.sum +++ b/go.sum @@ -190,10 +190,15 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= +github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= diff --git a/http/client.go b/http/client.go index 70c0639ec..8d3437bb5 100644 --- a/http/client.go +++ b/http/client.go @@ -8,18 +8,22 @@ import ( "fmt" "io" "io/ioutil" + "math" "math/rand" "net/http" "net/url" + "os" "path" "sort" "strconv" "strings" "time" + "github.com/hashicorp/go-retryablehttp" pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/ingest" + "github.com/molecula/featurebase/v2/logger" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/topology" "github.com/molecula/featurebase/v2/tracing" @@ -31,8 +35,11 @@ type InternalClient struct { defaultURI *pnet.URI serializer pilosa.Serializer + log logger.Logger + // The client to use for HTTP communication. - httpClient *http.Client + httpClient *http.Client + retryableClient *retryablehttp.Client // the local node's API, used for operations that we can short-circuit that way api *pilosa.API } @@ -40,7 +47,7 @@ type InternalClient struct { // NewInternalClient returns a new instance of InternalClient to connect to host. // If api is non-nil, the client uses it for some same-host operations instead // of going through http. -func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, error) { +func NewInternalClient(host string, remoteClient *http.Client, opts ...InternalClientOption) (*InternalClient, error) { if host == "" { return nil, pilosa.ErrHostRequired } @@ -50,16 +57,75 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, return nil, errors.Wrap(err, "getting URI") } - client := NewInternalClientFromURI(uri, remoteClient) + client := NewInternalClientFromURI(uri, remoteClient, opts...) return client, nil } -func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client) *InternalClient { - return &InternalClient{ +type InternalClientOption func(c *InternalClient) + +// WithClientRetryPeriod is the max amount of total time the client will +// retry failed requests using exponential backoff. +func WithClientRetryPeriod(period time.Duration) InternalClientOption { + min := time.Millisecond * 100 + + // do some math to figure out how many attempts we need to get our + // total sleep time close to the period + attempts := math.Log2(float64(period)) - math.Log2(float64(min)) + attempts += 0.3 // mmmm, fudge + if attempts < 1 { + attempts = 1 + } + fmt.Println("attempts: ", int(attempts)) + return func(c *InternalClient) { + rc := retryablehttp.NewClient() + rc.HTTPClient = c.httpClient + rc.RetryWaitMin = min + rc.RetryMax = int(attempts) + rc.CheckRetry = retryWith400Policy + c.retryableClient = rc + } +} + +func WithClientLogger(log logger.Logger) InternalClientOption { + return func(c *InternalClient) { + c.log = log + } +} + +func noRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { + return false, nil +} + +// retryWith400Policy wraps retryablehttp's default retry policy to +// also retry on 4XX errors which *should* be client errors and +// therefore useless to retry, but we have some incorrect status codes. +// TODO: fix the incorrect status codes so we can get rid of this. +func retryWith400Policy(ctx context.Context, resp *http.Response, err error) (bool, error) { + if resp != nil && resp.StatusCode >= 400 { + return true, nil + } + return retryablehttp.DefaultRetryPolicy(ctx, resp, err) +} + +func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client, opts ...InternalClientOption) *InternalClient { + ic := &InternalClient{ defaultURI: defaultURI, serializer: proto.Serializer{}, httpClient: remoteClient, + log: logger.NewStandardLogger(os.Stderr), } + + for _, opt := range opts { + opt(ic) + } + + if ic.retryableClient == nil { + rc := retryablehttp.NewClient() + rc.HTTPClient = ic.httpClient + rc.CheckRetry = noRetryPolicy + ic.retryableClient = rc + } + return ic } // MaxShardByIndex returns the number of shards on a server by index. @@ -1717,19 +1783,36 @@ func giveRawResponse(b bool) executeRequestOption { } } +type nopCloser struct { + *bytes.Reader +} + +func (n nopCloser) Close() error { + return nil +} + // executeRequest executes the given request and checks the Response. For // responses with non-2XX status, the body is read and closed, and an error is // returned. If the error is nil, the caller must ensure that the response body // is closed. func (c *InternalClient) executeRequest(req *http.Request, opts ...executeRequestOption) (*http.Response, error) { + return c.executeRetryableRequest(&retryablehttp.Request{Request: req}, opts...) +} + +func (c *InternalClient) executeRetryableRequest(req *retryablehttp.Request, opts ...executeRequestOption) (*http.Response, error) { + tracing.GlobalTracer.InjectHTTPHeaders(req.Request) + req.Close = false eo := &executeOpts{} for _, opt := range opts { opt(eo) } - tracing.GlobalTracer.InjectHTTPHeaders(req) - req.Close = false - resp, err := c.httpClient.Do(req) + resp, err := c.retryableClient.Do(req) + + return c.handleResponse(req.Request, eo, resp, err) +} + +func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp *http.Response, err error) (*http.Response, error) { if err != nil { if resp != nil { resp.Body.Close() @@ -2009,7 +2092,7 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, return resp.Body, nil } -func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error { +func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportIndexKeys") defer span.Finish() @@ -2026,14 +2109,14 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, ind url := fmt.Sprintf("%s/internal/translate/index/%s/%d", uri, index, partitionID) // Generate HTTP request. - httpReq, err := http.NewRequest("POST", url, rddbdata) + httpReq, err := retryablehttp.NewRequest("POST", url, readerFunc) if err != nil { return errors.Wrap(err, "creating request") } httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. - resp, err := c.executeRequest(httpReq.WithContext(ctx)) + resp, err := c.executeRetryableRequest(httpReq.WithContext(ctx)) if err != nil { return err } @@ -2041,7 +2124,7 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, ind return nil } -func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error { +func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportFieldKeys") defer span.Finish() @@ -2058,14 +2141,14 @@ func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, ind url := fmt.Sprintf("%s/internal/translate/field/%s/%s", uri, index, field) // Generate HTTP request. - httpReq, err := http.NewRequest("POST", url, rddbdata) + httpReq, err := retryablehttp.NewRequest("POST", url, readerFunc) if err != nil { return errors.Wrap(err, "creating request") } httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. - resp, err := c.executeRequest(httpReq.WithContext(ctx)) + resp, err := c.executeRetryableRequest(httpReq.WithContext(ctx)) if err != nil { return err } diff --git a/http/handler.go b/http/handler.go index e18a611c1..a1aa086c1 100644 --- a/http/handler.go +++ b/http/handler.go @@ -479,6 +479,9 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/internal/idalloc/data", handler.chkAuthN(handler.handleIDAllocData)).Methods("GET").Name("IDAllocData") router.HandleFunc("/internal/restore/{index}/{shardID}", handler.chkAuthN(handler.handlePostRestore)).Methods("POST").Name("Restore") + + router.HandleFunc("/internal/debug/rbf", handler.chkAuthN(handler.handleGetInternalDebugRBFJSON)).Methods("GET").Name("GetInternalDebugRBFJSON") + // endpoints for collecting cpu profiles from a chosen begin point to // when the client wants to stop. Used for profiling imports that // could be long or short. @@ -2207,6 +2210,18 @@ func validateProtobufHeader(r *http.Request) (error string, code int) { return } +// handleGetInternalDebugRBFJSON handles /internal/debug/rbf requests. +func (h *Handler) handleGetInternalDebugRBFJSON(w http.ResponseWriter, r *http.Request) { + buf, err := json.MarshalIndent(h.api.RBFDebugInfo(), "", " ") + if err != nil { + http.Error(w, "marshal json: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(buf) +} + // handleGetMetricsJSON handles /metrics.json requests, translating text metrics results to more consumable JSON. func (h *Handler) handleGetMetricsJSON(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { @@ -2723,14 +2738,17 @@ func (s queryValidationSpec) validate(query url.Values) error { return nil } -func GetHTTPClient(t *tls.Config) *http.Client { +type ClientOption func(client *http.Client, dialer *net.Dialer) *http.Client + +func GetHTTPClient(t *tls.Config, opts ...ClientOption) *http.Client { + dialer := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + } transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, + Proxy: http.ProxyFromEnvironment, + DialContext: dialer.DialContext, MaxIdleConns: 1000, MaxIdleConnsPerHost: 200, IdleConnTimeout: 90 * time.Second, @@ -2740,7 +2758,12 @@ func GetHTTPClient(t *tls.Config) *http.Client { if t != nil { transport.TLSClientConfig = t } - return &http.Client{Transport: transport} + + client := &http.Client{Transport: transport} + for _, opt := range opts { + client = opt(client, dialer) + } + return client } // handlePostImportAtomicRecord handles /import-atomic-record requests @@ -3493,7 +3516,7 @@ func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) { //validate shard for this node err = h.api.RestoreShard(ctx, indexName, shard, r.Body) if err != nil { - http.Error(w, fmt.Sprintf("failed to restore shared %v %v err:%v", indexName, shard, err), http.StatusBadRequest) + http.Error(w, fmt.Sprintf("failed to restore shard %v %v err:%v", indexName, shard, err), http.StatusBadRequest) return } diff --git a/http/handler_internal_test.go b/http/handler_internal_test.go index f4856e235..a073e7a38 100644 --- a/http/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -246,20 +246,20 @@ func TestAuthentication(t *testing.T) { GroupName: "Romantic Painters", } - validCV := authn.CookieValue{ + validCV := authn.AuthContext{ UserID: "snowstorm", UserName: "J.M.W. Turner", GroupMembership: []authn.Group{grp}, Token: &token, } - emptyCV := authn.CookieValue{ + emptyCV := authn.AuthContext{ UserID: "narcissus", UserName: "Caravaggio", GroupMembership: []authn.Group{}, Token: &token, } - expiredCV := authn.CookieValue{ + expiredCV := authn.AuthContext{ UserID: "narcissus", UserName: "Caravaggio", GroupMembership: []authn.Group{grp}, diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go index 2fca2f1ab..e23d10f8c 100644 --- a/internal/clustertests/cluster_test.go +++ b/internal/clustertests/cluster_test.go @@ -3,6 +3,8 @@ package clustertest import ( "context" + "fmt" + "net/http" "os" "os/exec" "testing" @@ -30,56 +32,53 @@ func TestClusterStuff(t *testing.T) { t.Fatalf("getting client: %v", err) } - t.Run("long pause", func(t *testing.T) { - err := cli1.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - err = cli1.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100}) - if err != nil { - t.Fatalf("creating field: %v", err) - } + if err := cli1.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{}); err != nil { + t.Fatalf("creating index: %v", err) + } + if err := cli1.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100}); err != nil { + t.Fatalf("creating field: %v", err) + } - req := &pilosa.ImportRequest{ - Index: "testidx", - Field: "testf", - } - req.ColumnIDs = make([]uint64, 10) - req.RowIDs = make([]uint64, 10) + req := &pilosa.ImportRequest{ + Index: "testidx", + Field: "testf", + } + req.ColumnIDs = make([]uint64, 10) + req.RowIDs = make([]uint64, 10) - for i := 0; i < 1000; i++ { - req.RowIDs[i%10] = 0 - req.ColumnIDs[i%10] = uint64((i/10)*pilosa.ShardWidth + i%10) - req.Shard = uint64(i / 10) - if i%10 == 9 { - err = cli1.Import(context.Background(), nil, req, &pilosa.ImportOptions{}) - if err != nil { - t.Fatalf("importing: %v", err) - } - } - } - - // Check query results from each node. - for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} { - r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) + for i := 0; i < 1000; i++ { + req.RowIDs[i%10] = 0 + req.ColumnIDs[i%10] = uint64((i/10)*pilosa.ShardWidth + i%10) + req.Shard = uint64(i / 10) + if i%10 == 9 { + err = cli1.Import(context.Background(), nil, req, &pilosa.ImportOptions{}) if err != nil { - t.Fatalf("count querying pilosa%d: %v", i, err) - } - if r.Results[0].(uint64) != 1000 { - t.Fatalf("count on pilosa%d after import is %d", i, r.Results[0].(uint64)) + t.Fatalf("importing: %v", err) } } + } + + // Check query results from each node. + for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} { + r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) + if err != nil { + t.Fatalf("count querying pilosa%d: %v", i, err) + } + if r.Results[0].(uint64) != 1000 { + t.Fatalf("count on pilosa%d after import is %d", i, r.Results[0].(uint64)) + } + } + t.Run("long pause", func(t *testing.T) { pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s") pcmd.Stdout = os.Stdout pcmd.Stderr = os.Stderr t.Log("pausing pilosa3 for 10s") - err = pcmd.Start() - if err != nil { + + if err := pcmd.Start(); err != nil { t.Fatalf("starting pumba command: %v", err) } - err = pcmd.Wait() - if err != nil { + if err := pcmd.Wait(); err != nil { t.Fatalf("waiting on pumba pause cmd: %v", err) } @@ -98,6 +97,89 @@ func TestClusterStuff(t *testing.T) { } } }) + + t.Run("backup", func(t *testing.T) { + // do backup with node 1 down, but restart it after a few seconds + if err := sendCmd("docker", "stop", "clustertests_pilosa1_1"); err != nil { + t.Fatalf("sending stop command: %v", err) + } + var backupCmd *exec.Cmd + tmpdir := t.TempDir() + if backupCmd, err = startCmd( + "featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest")); err != nil { + t.Fatalf("sending backup command: %v", err) + } + time.Sleep(time.Second * 5) + if err = sendCmd("docker", "start", "clustertests_pilosa1_1"); err != nil { + t.Fatalf("sending start command: %v", err) + } + + if err = backupCmd.Wait(); err != nil { + t.Fatalf("waiting on backup to finish: %v", err) + } + + fmt.Println("STARTING RESTORE") + + client := http.Client{} + if req, err := http.NewRequest(http.MethodDelete, "http://pilosa1:10101/index/testidx", nil); err != nil { + t.Fatalf("getting req: %v", err) + } else if resp, err := client.Do(req); err != nil { + t.Fatalf("doing request: %v", err) + } else if resp.StatusCode >= 400 { + t.Fatalf("bad response: %v", resp) + } + + var restoreCmd *exec.Cmd + if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101"); err != nil { + t.Fatalf("starting restore: %v", err) + } + time.Sleep(time.Millisecond * 50) + if err = sendCmd("docker", "stop", "clustertests_pilosa2_1"); err != nil { + t.Fatalf("sending stop command: %v", err) + } + + time.Sleep(time.Second * 10) + if err = sendCmd("docker", "start", "clustertests_pilosa2_1"); err != nil { + t.Fatalf("sending stop command: %v", err) + } + if err := restoreCmd.Wait(); err != nil { + t.Fatalf("restore failed: %v", err) + } + + // now do backup with all nodes down and too short a timeout + // so it fails. Has be to be all 3 because the cluster has + // replicas=3 and the backup command will retry on replicas. + if backupCmd, err = startCmd( + "featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms"); err != nil { + t.Fatalf("sending second backup command: %v", err) + } + time.Sleep(time.Millisecond * 10) // want the backup to get started, then fail + if err = sendCmd("docker", "stop", "clustertests_pilosa1_1"); err != nil { + t.Fatalf("sending stop command: %v", err) + } + if err = sendCmd("docker", "stop", "clustertests_pilosa2_1"); err != nil { + t.Fatalf("sending stop command: %v", err) + } + if err = sendCmd("docker", "stop", "clustertests_pilosa3_1"); err != nil { + t.Fatalf("sending stop command: %v", err) + } + + time.Sleep(time.Second * 5) + + if err = sendCmd("docker", "start", "clustertests_pilosa1_1"); err != nil { + t.Fatalf("sending start command: %v", err) + } + if err = sendCmd("docker", "start", "clustertests_pilosa2_1"); err != nil { + t.Fatalf("sending start command: %v", err) + } + if err = sendCmd("docker", "start", "clustertests_pilosa3_1"); err != nil { + t.Fatalf("sending start command: %v", err) + } + if err = backupCmd.Wait(); err == nil { + t.Fatal("backup command should have errored but didn't") + } + + }) } func waitForStatus(t *testing.T, stator func(context.Context) (string, error), status string, n int, sleep time.Duration) { diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml index 4192be6f8..46143a3f6 100644 --- a/internal/clustertests/docker-compose.yml +++ b/internal/clustertests/docker-compose.yml @@ -9,6 +9,7 @@ services: - "33455:10101" environment: - PILOSA_NAME=pilosa1 + - PILOSA_ETCD_DIR=/root/.etcd - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201 - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa1:10201 - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 @@ -28,6 +29,7 @@ services: - "33456:10101" environment: - PILOSA_NAME=pilosa2 + - PILOSA_ETCD_DIR=/root/.etcd - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201 - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa2:10201 - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 @@ -47,6 +49,7 @@ services: - "33457:10101" environment: - PILOSA_NAME=pilosa3 + - PILOSA_ETCD_DIR=/root/.etcd - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201 - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa3:10201 - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go index 1cfa81417..256078a90 100644 --- a/internal/clustertests/pause_node_test.go +++ b/internal/clustertests/pause_node_test.go @@ -23,11 +23,16 @@ import ( "github.com/pkg/errors" ) -func sendCmd(cmd string, args ...string) error { +func startCmd(cmd string, args ...string) (*exec.Cmd, error) { pcmd := exec.Command(cmd, args...) pcmd.Stdout = os.Stdout pcmd.Stderr = os.Stderr err := pcmd.Start() + return pcmd, err +} + +func sendCmd(cmd string, args ...string) error { + pcmd, err := startCmd(cmd, args...) if err != nil { return errors.Wrap(err, "starting cmd") } diff --git a/lattice/src/services/__mocks__/eventServices.tsx b/lattice/src/services/__mocks__/eventServices.tsx new file mode 100644 index 000000000..a6203b244 --- /dev/null +++ b/lattice/src/services/__mocks__/eventServices.tsx @@ -0,0 +1,12 @@ +const pilosa = { + get: { + auth() { + return new Promise((resolve, reject) => {}); + }, + userinfo() { + return new Promise((resolve, reject) => {}); + }, + }, +}; + +module.exports.pilosa = pilosa; diff --git a/lattice/src/services/useAuth.test.tsx b/lattice/src/services/useAuth.test.tsx new file mode 100644 index 000000000..2cf784786 --- /dev/null +++ b/lattice/src/services/useAuth.test.tsx @@ -0,0 +1,96 @@ +import { AxiosResponse } from 'axios'; +import { act } from 'react-dom/test-utils'; +import ReactDOM from 'react-dom'; + +import { ProvideAuth, useAuth } from 'services/useAuth'; +import { pilosa } from './eventServices'; + +jest.mock('./eventServices'); + +const AUTHENTICATED = 'Authenticated'; +const NOTAUTHED = 'Not Authed'; +const AUTHOFF = 'Auth off'; + +function TestUseAuthComponent() { + const auth = useAuth(); + + if (auth.isAuthOn === true && auth.isAuthenticated === true) { + return
{AUTHENTICATED}
; + } else if (auth.isAuthOn === true && auth.isAuthenticated === false) { + return
{NOTAUTHED}
; + } else { + return
{AUTHOFF}
; + } +} + +beforeEach(() => { + jest.clearAllMocks(); +}); + +test('test useAuth - expect authenticated', async () => { + const mockResponse: AxiosResponse = { + status: 200, + data: 'OK', + statusText: '', + headers: {}, + config: {}, + }; + const root = document.createElement('root'); + await act(async () => { + jest.spyOn(pilosa.get, 'auth').mockResolvedValueOnce(mockResponse); + ReactDOM.render( + + + , + root + ); + }); + expect(pilosa.get.auth).toHaveBeenCalledTimes(1); + expect(root.innerHTML).toContain(AUTHENTICATED); +}); + +test('test useAuth - expect not authed', async () => { + const mockResponse: AxiosResponse = { + status: 200, + data: '', + statusText: '', + headers: {}, + config: {}, + }; + + const root = document.createElement('root'); + await act(async () => { + jest.spyOn(pilosa.get, 'auth').mockResolvedValueOnce(mockResponse); + ReactDOM.render( + + + , + root + ); + }); + expect(pilosa.get.auth).toHaveBeenCalledTimes(1); + expect(root.innerHTML).toContain(NOTAUTHED); +}); + +test('test useAuth - expect auth off', async () => { + const mockResponse: AxiosResponse = { + status: 204, + data: '', + statusText: '', + headers: {}, + config: {}, + }; + + const root = document.createElement('root'); + await act(async () => { + jest.spyOn(pilosa.get, 'auth').mockResolvedValueOnce(mockResponse); + ReactDOM.render( + + + , + root + ); + }); + expect(pilosa.get.auth).toHaveBeenCalledTimes(1); + expect(root.innerHTML).toContain(AUTHOFF); +}); diff --git a/qa/scripts/cloud-init.sh b/qa/scripts/cloud-init.sh deleted file mode 100755 index 8d710d0b2..000000000 --- a/qa/scripts/cloud-init.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -ex -# generate log -exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 - -# Install packages -yum update -y -yum install postgresql -y - -# Configure host system -echo 'cat /proc/sys/fs/file-max' -sysctl -w fs.file-max=262144 -sysctl -p -echo 'cat /proc/sys/fs/file-max' - -# yum install golang -y # latest verion in ec2 is 1.15.14 -# install go 1.16.9 manually -curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz -tar xvf go1.16.10.linux-amd64.tar.gz -chown -R root:root ./go -mv go /usr/local -echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null -source /etc/profile - -# install aws session manager pluggin -curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" -yum install -y session-manager-plugin.rpm diff --git a/qa/scripts/configureFeatureBase.json b/qa/scripts/configureFeatureBase.json deleted file mode 100644 index 6afd135d9..000000000 --- a/qa/scripts/configureFeatureBase.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Parameters": { - "commands": [ - "#!/bin/bash", - "mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase", - "mv /home/ec2-user/featurebase.conf /etc/", - "mv /home/ec2-user/featurebase.service /etc/systemd/system/", - "adduser molecula", - "sudo mkdir /var/log/molecula", - "sudo chown molecula /var/log/molecula", - "sudo mkdir -p /opt/molecula/featurebase", - "sudo chown molecula /opt/molecula/featurebase", - "systemctl daemon-reload", - "sudo systemctl start featurebase", - "sudo systemctl enable featurebase", - "sudo systemctl status featurebase", - "curl localhost:10101" - ] - } -} \ No newline at end of file diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh deleted file mode 100755 index b928a31e7..000000000 --- a/qa/scripts/deployNode.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -# To run script: ./deployNode.sh $PROFILE - -function deploy_node() { - # get AMI, security group and subnet ID - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - if [[ $? > 0 ]]; then - echo "aws session manager failed to find AMI" - exit 1 - fi - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - if [[ $? > 0 ]]; then - echo "aws session manager failed to find security group" - exit 1 - fi - - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - if [[ $? > 0 ]]; then - echo "aws session manager failed to find subnet ID" - exit 1 - fi - - # launch EC2 instance and get instance ID - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - if [[ $? > 0 ]]; then - echo "aws run-instances failed to launch a new EC2 instance" - exit 1 - fi - - INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"') - echo "aws run-instances succeeded in launching a new EC2 instance with instance ID: " $INSTANCE_ID -} - -function initialize_featurebase() { - # get IP for node - for i in {0..24} - do - IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - if [ -n "$IP" ]; then - echo "Public IP for EC2 instance found: " $IP - break - fi - - if [[ $? > 0 ]]; then - echo "aws cli describe-instances command failed to find public IP" - terminate_node - exit 1 - fi - - sleep 5 - done - - sleep 60 # to allow enough time for node to be ready for use - - # copy featurebase binary and files to ec2 instance - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. - if [[ $? > 0 ]]; then - echo "scp of featurebase binary, service and config files to EC2 instance failed" - terminate_node - exit 1 - fi - - # execute script to configure featurebase on the EC2 node - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region $REGION - if [[ $? > 0 ]]; then - echo "aws cli session manager send-command failed" - terminate_node - exit 1 - fi -} - -function terminate_node() { - aws ec2 terminate-instances --instance-ids $INSTANCE_ID --profile $PROFILE -} - -# Pass variables to shell script -PROFILE=$1 -shift - -# set some variables -INSTANCE="t3a.large" -REGION="us-east-2" - -# get AMI, security group and subnet for EC2 instance, -# launch instance, save instance Id and run cloud-init to set up node env -deploy_node - -# Get IP for instance, scp featurebase binary, config and service files; -# set up featurebase config in node -initialize_featurebase - -# terminate node -terminate_node diff --git a/qa/scripts/featurebase.conf b/qa/scripts/featurebase.conf deleted file mode 100644 index 7716d0f6b..000000000 --- a/qa/scripts/featurebase.conf +++ /dev/null @@ -1,30 +0,0 @@ -name = "pilosa1" -bind = "0.0.0.0:10101" -bind-grpc = "0.0.0.0:20101" - -data-dir = "/opt/molecula/featurebase" -log-path = "/var/log/molecula/featurebase.log" - -max-file-count=900000 -max-map-count=900000 - -long-query-time = "10s" - -[postgres] - - bind = "localhost:55432" - -[cluster] - - name = "cluster1" - replicas = 1 - -[etcd] - - listen-client-address = "http://localhost:10401" - listen-peer-address = "http://localhost:10301" - initial-cluster = "pilosa1=http://localhost:10301" - -[metric] - - service = "prometheus" \ No newline at end of file diff --git a/qa/scripts/featurebase.service b/qa/scripts/featurebase.service deleted file mode 100644 index a543b92af..000000000 --- a/qa/scripts/featurebase.service +++ /dev/null @@ -1,13 +0,0 @@ -# Not Ansible managed - -[Unit] -Description="Service for FeatureBase" - -[Service] -RestartSec=30 -Restart=on-failure -EnvironmentFile= -User=molecula -ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf - -[Install] \ No newline at end of file diff --git a/qa/scripts/runSamsungGauntlet.sh b/qa/scripts/runSamsungGauntlet.sh new file mode 100644 index 000000000..85fd22c67 --- /dev/null +++ b/qa/scripts/runSamsungGauntlet.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./setupSamsungGauntlet.sh +./testSamsungGauntlet.sh +./teardownSamsungGauntlet.sh diff --git a/qa/scripts/runSmokeTest.sh b/qa/scripts/runSmokeTest.sh new file mode 100644 index 000000000..554a234cc --- /dev/null +++ b/qa/scripts/runSmokeTest.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +./setupSmokeTest.sh +./testSmokeTest.sh +./teardownSmokeTest.sh diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh new file mode 100755 index 000000000..1c741be78 --- /dev/null +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# To run script: ./setupSamsungGauntlet.sh +# requires TF_VAR_gitlab_token env var to be set + +pushd ./qa/tf/gauntlet/samsung +export TF_IN_AUTOMATION=1 +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > outputs.json +popd + +# get the bastion host +BASTION=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init +#while true +#do +# nc -G 2 -w 1 $BASTION 22 +# if [ $? -eq 0 ] +# then +# break +# fi +#done +sleep 150 + +# verify featurebase running +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" +if (( $? != 0 )) +then + echo "Featurebase cluster not running" + exit 1 +fi + + diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh new file mode 100755 index 000000000..766d9a82b --- /dev/null +++ b/qa/scripts/setupSmokeTest.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# To run script: ./setupSmokeTest.sh +# requires TF_VAR_gitlab_token env var to be set + +pushd ./qa/tf/ci/smoketest +export TF_IN_AUTOMATION=1 +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > outputs.json +popd + +# get the bastion host +BASTION=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init +echo "Waiting for cluster to become available..." +# jaffee - I do wanna do a loop here, but I give up, and am running home to sleep... -POK +sleep 150 + +# verify featurebase running +echo "Verifying featurebase cluster running..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no ec2-user@${BASTION} "curl -s http://${NODE}:10101/status" +if (( $? != 0 )) +then + echo "Featurebase cluster not running" + exit 1 +fi + +echo "Cluster running." + + diff --git a/qa/scripts/teardownSamsungGauntlet.sh b/qa/scripts/teardownSamsungGauntlet.sh new file mode 100755 index 000000000..ae614d99b --- /dev/null +++ b/qa/scripts/teardownSamsungGauntlet.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# To run script: ./teardownSamsungGauntlet.sh +# requires TF_VAR_gitlab_token env var to be set + +cd qa/tf/gauntlet/samsung +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/teardownSmokeTest.sh b/qa/scripts/teardownSmokeTest.sh new file mode 100755 index 000000000..1e67d2895 --- /dev/null +++ b/qa/scripts/teardownSmokeTest.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# To run script: ./teardownSmokeTest.sh +# requires TF_VAR_gitlab_token env var to be set + +cd qa/tf/ci/smoketest +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/testSamsungGauntlet.sh b/qa/scripts/testSamsungGauntlet.sh new file mode 100755 index 000000000..efe0b8951 --- /dev/null +++ b/qa/scripts/testSamsungGauntlet.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# get the bastion host +BASTION=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/gauntlet/samsung/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +# generate csv files +GOOS=linux GOARCH=arm64 go build ./qa/simulacraData/... +scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${BASTION}:/data +if (( $? != 0 )) +then + echo "Copy failed" + exit 1 +fi + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "cd /data && /data/simulacraData" +if (( $? != 0 )) +then + echo "Making big files failed" + exit 1 +fi + +# ingest these files the way that samsung does it +scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${BASTION}: +if (( $? != 0 )) +then + echo "Copy ingest script failed" + exit 1 +fi + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 1" +if (( $? != 0 )) +then + echo "Running 1 testSamsungPayload.sh failed" + exit 1 +fi + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 0" +if (( $? != 0 )) +then + echo "Running 0 testSamsungPayload.sh failed" + exit 1 +fi + +# query workload that runs \ No newline at end of file diff --git a/qa/scripts/ingestWorkload.sh b/qa/scripts/testSamsungPayload.sh similarity index 91% rename from qa/scripts/ingestWorkload.sh rename to qa/scripts/testSamsungPayload.sh index 23d11dd32..8c36c63dc 100755 --- a/qa/scripts/ingestWorkload.sh +++ b/qa/scripts/testSamsungPayload.sh @@ -1,7 +1,14 @@ #!/usr/bin/env bash +# path for featurebase binary +FEATUREBASE_PATH=/usr/local/bin + +# path for directory with csv directory files for all fields to be ingested +CSV_DIR_PATH=/data + + # To run: -# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag} +# ./testSamsungPayload.sh {Local host & port for featurebase} {initialize flag} function delete_field { if (($INITIALIZE == 0)); @@ -30,18 +37,10 @@ function ingest_set_field { $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE } -# path for featurebase binary -FEATUREBASE_PATH=$1 -shift - # featurebase host & port HOST=$1 shift -# path for directory with csv directory files for all fields to be ingested -CSV_DIR_PATH=$1 -shift - # intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing INITIALIZE=$1 shift diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh new file mode 100755 index 000000000..ed748cf97 --- /dev/null +++ b/qa/scripts/testSmokeTest.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# get the bastion host +BASTION=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using bastion ${BASTION}" + +NODE=$(cat ./qa/tf/ci/smoketest/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using node ${NODE}" + +echo "Copying tests to remote" +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/testcases/smoketest/*.py ec2-user@${BASTION}:/data +if (( $? != 0 )) +then + echo "Copy failed" + exit 1 +fi + +# run smoke test +echo "Running smoke test..." +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} " pushd /data; pytest --junitxml=report.xml; popd" +if (( $? != 0 )) +then + echo "Unable to run smoke test" + exit 1 +fi + +echo "Copying test report to local" +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION}:/data/report.xml report.xml +if (( $? != 0 )) +then + echo "Copy failed" + exit 1 +fi + +echo "Smoke test complete" \ No newline at end of file diff --git a/qa/testcases/smoketest/test_smoke.py b/qa/testcases/smoketest/test_smoke.py new file mode 100644 index 000000000..ba45124ff --- /dev/null +++ b/qa/testcases/smoketest/test_smoke.py @@ -0,0 +1,7 @@ +# content of test_smoke.py +def inc(x): + return x + 1 + + +def test_answer(): + assert inc(3) == 5 \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/README.md b/qa/tf/.modules/featurebase-cluster/README.md new file mode 100644 index 000000000..d1525adba --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/README.md @@ -0,0 +1,37 @@ +# Summary + +This module provisions a VPC, subnets, instances, keys, and security groups needed for a basic featurebase cluster running in AWS. It is meant to be used as a module. For example: + +```hcl +module "featurebase" { + source "/path/to/module/" + cluster_prefix = "sprockets" + azs = ["us-east-1a", "us-east-1b", "us-east-1c"] +} +``` + +The path to the module is wherever the `featurebase-cloud` directory is. So if you have put it in `/var/opt/terraform/modules/featurebase-cloud` then calling the module would look like: + +```hcl +module "featurebase" { + source "/var/opt/terraform/modules/featurebase-cloud" + cluster_prefix = "sprockets" +} +``` + +Much more is configurable; for a complete list, look in `variables.tf`. Reasonable defaults have been set. + +## AWS Access + +Please make sure you have set up your AWS access in either environment variables, or in the credentials file. + +Some useful links for this are: + +AWS Environment Variables + +## State + +State is currently kept locally, for as this is intended for PoCs. It can be stored in a remote s3 or GCS bucket if desired. + + + \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/main.tf b/qa/tf/.modules/featurebase-cluster/main.tf new file mode 100644 index 000000000..133226e75 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/main.tf @@ -0,0 +1,229 @@ +data "aws_ami" "amazon_linux_2" { + most_recent = true + owners = ["amazon"] + filter { + name = "name" + values = ["amzn2-ami-hvm-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "architecture" + values = ["arm64"] + } +} + +resource "aws_instance" "fb_cluster_nodes" { + count = var.fb_data_node_count + ami = data.aws_ami.amazon_linux_2.id + instance_type = var.fb_data_node_type + key_name = aws_key_pair.gitlab-featurebase-ci.key_name + vpc_security_group_ids = [aws_security_group.featurebase.id] + monitoring = true + subnet_id = var.subnet != "" ? var.subnet : module.vpc.private_subnets[count.index % length(module.vpc.private_subnets)] + availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)] + iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}" + + root_block_device { + volume_type = "gp3" + volume_size = 20 + } + + ebs_block_device { + device_name = "/dev/sdb" + volume_type = var.fb_data_disk_type + volume_size = var.fb_data_disk_size_gb + iops = var.fb_data_disk_iops + } + + tags = { + Prefix = "${var.cluster_prefix}" + Name = "${var.cluster_prefix}-featurebase-cluster-${count.index}" + Role = "cluster_node" + } + + user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region })) +} + +resource "aws_instance" "fb_ingest" { + count = var.fb_ingest_node_count + ami = data.aws_ami.amazon_linux_2.id + key_name = aws_key_pair.gitlab-featurebase-ci.key_name + vpc_security_group_ids = [aws_security_group.ingest.id] + instance_type = var.fb_ingest_type + associate_public_ip_address = true + monitoring = true + subnet_id = var.subnet != "" ? var.subnet : module.vpc.public_subnets[count.index % length(module.vpc.public_subnets)] + availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)] + iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}" + + root_block_device { + volume_type = "gp3" + volume_size = 20 + } + + ebs_block_device { + device_name = "/dev/sdb" + volume_type = var.fb_ingest_disk_type + volume_size = var.fb_ingest_disk_size_gb + iops = var.fb_ingest_disk_iops + } + + tags = { + Prefix = "${var.cluster_prefix}" + Name = "${var.cluster_prefix}-featurebase-ingest-${count.index}" + Role = "ingest_node" + } + + user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, branch = var.branch, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region })) +} + +resource "aws_key_pair" "gitlab-featurebase-ci" { + key_name = "gitlab-featurebase-ci" + public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL" +} + +resource "aws_security_group" "featurebase" { + name = "allow_featurebase" + description = "Allow featurebase inbound traffic" + vpc_id = module.vpc.vpc_id + + ingress { + description = "TLS from Internal" + from_port = 10101 + to_port = 10101 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + + description = "GRPC from Internal" + from_port = 20101 + to_port = 20101 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + description = "PostgreSQL from Internal" + from_port = 55432 + to_port = 55432 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + description = "etcd from internal" + from_port = 10301 + to_port = 10301 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + description = "etcd from internal 2" + from_port = 10401 + to_port = 10401 + protocol = "tcp" + cidr_blocks = [module.vpc.vpc_cidr_block] + } + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + tags = { + Name = "allow_featurebase" + } +} + +resource "aws_security_group" "ingest" { + name = "allow_ingest" + description = "Allow ingest inbound traffic" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 10101 + to_port = 10101 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + tags = { + Name = "allow_ingest" + } +} + +resource "aws_iam_instance_profile" "fb_cluster_node_profile" { + name = "fb_cluster_node_profile" + role = aws_iam_role.fb_cluster_node_role.name +} + +resource "aws_iam_role" "fb_cluster_node_role" { + name = "fb_cluster_node" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = "ec2.amazonaws.com" + } + }, + ] + }) + + inline_policy { + name = "ec2_read_all" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = ["ec2:Describe*"] + Effect = "Allow" + Resource = "*" + }, + ] + }) + } + +} \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/outputs.tf b/qa/tf/.modules/featurebase-cluster/outputs.tf new file mode 100644 index 000000000..e52e7344c --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/outputs.tf @@ -0,0 +1,7 @@ +output "ingest_ips" { + value = aws_instance.fb_ingest.*.public_ip +} + +output "data_node_ips" { + value = aws_instance.fb_cluster_nodes.*.private_ip +} \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/provider.tf b/qa/tf/.modules/featurebase-cluster/provider.tf new file mode 100644 index 000000000..cf53a4ed7 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/provider.tf @@ -0,0 +1,11 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.38.0" + } + } +} + diff --git a/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl new file mode 100644 index 000000000..9a7e71a93 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl @@ -0,0 +1,188 @@ +#!/bin/bash + +#path to the featurebase.conf file +CONFIG_FILE_PATH="/etc/featurebase.conf" +#path to the featurebase.service file +SERVICE_FILE_PATH="/etc/systemd/system/featurebase.service" + +AWS_INSTANCE_ID="" +#IP of this node +PRIVATE_IP="" +PRIVATE_IP_INDEX=-1 +#IPs of the cluster +CLUSTER_IPS="" + +get_aws_instance_id() { + echo "Getting AWS instance ID..." + while true + do + curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null + if [ $? -eq 0 ] + then + break + fi + done + AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id` + echo "AWS instance ID is: $${AWS_INSTANCE_ID}" +} + +wait_on_all_cluster_ips() { + echo "Waiting on all cluster IPs..." + # get IP for node + IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region}) + IP_LENGTH=`echo "$IPS" | wc -l` + + for i in {0..24} + do + echo "Comparing $${IP_LENGTH} with ${node_count}" + if [ $IP_LENGTH == "${node_count}" ]; then + echo "Cluster is up after $${i} tries." + break + fi + sleep 10s + done + + if [ $IP_LENGTH != "${node_count}" ]; then + echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes." + exit 1 + fi +} + +get_private_ip() { + echo "Getting private IP address..." + PRIVATE_IP=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=instance-id,Values=$${AWS_INSTANCE_ID}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region}) + echo "Private IP is $${PRIVATE_IP}" +} + +get_cluster_ips() { + echo "Getting cluster IPs..." + # get IP for node + IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region}) + IP_LENGTH=`echo "$IPS" | wc -l` + + IFS=$'\n' + cnt=0 + for ip in $IPS + do + echo $cnt $ip + if (($cnt + 1 != $IP_LENGTH)) + then + CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301," + else + CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301" + fi + echo "comparing $ip to $PRIVATE_IP" + if [ "$ip" = "$PRIVATE_IP" ]; then + PRIVATE_IP_INDEX=$cnt + fi + cnt=$((cnt+1)) + done + + echo "CLUSTER_IPS are: $${CLUSTER_IPS}" +} + +write_featurebase_config_file() { + echo "Writing featurebase.conf file..." + cat << EOT > $${CONFIG_FILE_PATH} +name = "p$${PRIVATE_IP_INDEX}" +bind = "0.0.0.0:10101" +bind-grpc = "0.0.0.0:20101" + +data-dir = "/data/featurebase" +log-path = "/var/log/molecula/featurebase.log" + +max-file-count=900000 +max-map-count=900000 + +long-query-time = "10s" + +[postgres] + + bind = "localhost:55432" + +[cluster] + + name = "${cluster_prefix}" + replicas = ${fb_cluster_replica_count} + +[etcd] + + listen-client-address = "http://$${PRIVATE_IP}:10401" + listen-peer-address = "http://$${PRIVATE_IP}:10301" + initial-cluster = "$${CLUSTER_IPS}" + +[metric] + + service = "prometheus" +EOT + + echo "featurebase.conf written to $${CONFIG_FILE_PATH}." +} + +write_featurebase_service_file() { + echo "Writing featurebase.service file..." + cat << EOT > $${SERVICE_FILE_PATH} +# Not Ansible managed + +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +EOT + + echo "featurebase.service written to $${SERVICE_FILE_PATH}." + +} + +#get the instance id +get_aws_instance_id + +#copy the script so we can look at it later if needed +sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt /home/ec2-user/setup_cluster_node.sh + +#wait for the count of nodes to equal requested nodes +wait_on_all_cluster_ips + +#get private ip +get_private_ip + +#generate cluster ips +get_cluster_ips + +#write the featurebase config file +write_featurebase_config_file + +#write the featurebase service file +write_featurebase_service_file + +#get the featurebase binary and put in in the right spot +echo "Getting featurebase binary..." +curl --fail --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64 +chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64" +chmod ugo+x "/home/ec2-user/featurebase_linux_arm64" + +mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase +echo "featurebase binary copied." + +sudo mkdir /data +sudo mkfs.ext4 /dev/nvme1n1 +sudo mount /dev/nvme1n1 /data + +adduser molecula +sudo mkdir /var/log/molecula +sudo chown molecula /var/log/molecula +sudo mkdir -p /data/featurebase +sudo chown molecula /data/featurebase +sudo systemctl daemon-reload +sudo systemctl start featurebase +sudo systemctl enable featurebase +sudo systemctl status featurebase + +echo "Done!" \ No newline at end of file diff --git a/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl new file mode 100644 index 000000000..8c032f0e5 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl @@ -0,0 +1,72 @@ +#!/bin/bash + +AWS_INSTANCE_ID="" + +get_aws_instance_id() { + echo "Getting AWS instance ID..." + while true + do + curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null + if [ $? -eq 0 ] + then + break + fi + done + AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id` + echo "AWS instance ID is: $${AWS_INSTANCE_ID}" +} + +wait_on_all_ingest_ips() { + echo "Waiting on all cluster IPs..." + # get IP for node + IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=ingest_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region}) + IP_LENGTH=`echo "$IPS" | wc -l` + + for i in {0..24} + do + echo "Comparing $${IP_LENGTH} with ${node_count}" + if [ $IP_LENGTH == "${node_count}" ]; then + echo "Cluster is up after $${i} tries." + break + fi + sleep 10s + done + + if [ $IP_LENGTH != "${node_count}" ]; then + echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes." + exit 1 + fi +} + +#copy the script so we can look at it later if needed +sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt ~/setup_ingest_node.sh + +#get the instance id +get_aws_instance_id + +#wait for the count of nodes to equal requested nodes +wait_on_all_ingest_ips + +echo "Getting featurebase binary..." +curl --fail --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/${branch}/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64 +chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64" +chmod ugo+x "/home/ec2-user/featurebase_linux_arm64" + +mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase +echo "featurebase binary copied." + + +sudo mkdir /data +sudo mkfs.ext4 /dev/nvme1n1 +sudo mount /dev/nvme1n1 /data + +sudo chown -R ec2-user /data + +echo "Installing pytest" +pip3 install -U pytest +pip3 install -U requests + +echo "Done." + + + diff --git a/qa/tf/.modules/featurebase-cluster/variables.tf b/qa/tf/.modules/featurebase-cluster/variables.tf new file mode 100644 index 000000000..dcb159820 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/variables.tf @@ -0,0 +1,98 @@ +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} + +variable "fb_ingest_type" { + type = string + default = "c6g.2xlarge" +} + +variable "fb_ingest_node_count" { + type = number + default = 1 +} + +variable "fb_data_node_type" { + type = string + default = "c6g.16xlarge" +} + +variable "fb_data_node_count" { + type = number + default = 3 +} + +variable "fb_cluster_replica_count" { + type = number + default = 1 +} + +variable "subnet" { + default = "" +} + +variable "zone" { + default = "" +} + +variable "fb_data_disk_type" { + default = "gp3" +} +variable "fb_data_disk_iops" { + default = 1000 +} + +variable "fb_data_disk_size_gb" { + default = 100 +} + +variable "fb_ingest_disk_type" { + default = "gp3" +} +variable "fb_ingest_disk_iops" { + default = 1000 +} + +variable "fb_ingest_disk_size_gb" { + default = 100 +} + +variable "azs" { + type = list(any) + default = ["us-east-2a", "us-east-2b", "us-east-2c"] +} + +variable "private_subnets" { + type = list(any) + default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] +} + +variable "public_subnets" { + type = list(any) + default = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"] +} + +variable "vpc_cidr" { + default = "10.0.0.0/16" +} + +variable "region" { + description = "Region to create AWS resources in" + type = string +} + +variable "profile" { + description = "Profile to use to authenticate with AWS" + type = string +} + +variable "gitlab_token" { + description = "Gitlab API token" + type = string +} + +variable "branch" { + description = "The branch we are on" + type = string +} diff --git a/qa/tf/.modules/featurebase-cluster/vpc.tf b/qa/tf/.modules/featurebase-cluster/vpc.tf new file mode 100644 index 000000000..4a09c9275 --- /dev/null +++ b/qa/tf/.modules/featurebase-cluster/vpc.tf @@ -0,0 +1,16 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + + name = "${var.cluster_prefix}" + cidr = var.vpc_cidr + azs = var.azs + private_subnets = var.private_subnets + public_subnets = var.public_subnets + + enable_nat_gateway = true + enable_vpn_gateway = false + + tags = { + Name = "${var.cluster_prefix}" + } +} \ No newline at end of file diff --git a/qa/tf/README.md b/qa/tf/README.md new file mode 100644 index 000000000..7117ebf3c --- /dev/null +++ b/qa/tf/README.md @@ -0,0 +1,16 @@ +# Deploy testing environments with this one weird trick! + +This directory contains Terraform to deploy test environments both ad-hoc and as part of CI/CD pipelines. + +The .modules contains the guts of the operation, the things you probably want are in the other directories, each with a README. + +## How to Terraform + +With terraform installed (`brew install terraform` if not)... + +You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down. + +## Other prerequisites: +Please read these carefully. + + diff --git a/qa/tf/ci/smoketest/main.tf b/qa/tf/ci/smoketest/main.tf new file mode 100644 index 000000000..80753a1f9 --- /dev/null +++ b/qa/tf/ci/smoketest/main.tf @@ -0,0 +1,11 @@ + +module "ci-cluster" { + source = "../../.modules/featurebase-cluster" + cluster_prefix = var.cluster_prefix + region = var.region + profile = var.profile + fb_data_node_type = "m6g.large" + fb_data_node_count = 1 + gitlab_token = var.gitlab_token + branch = var.branch +} diff --git a/qa/tf/ci/smoketest/outputs.tf b/qa/tf/ci/smoketest/outputs.tf new file mode 100644 index 000000000..adcc96dc9 --- /dev/null +++ b/qa/tf/ci/smoketest/outputs.tf @@ -0,0 +1,9 @@ +output "ingest_ips" { + description = "List of ingest IPs" + value = module.ci-cluster.ingest_ips +} + +output "data_node_ips" { + description = "List of data node IPs" + value = module.ci-cluster.data_node_ips +} \ No newline at end of file diff --git a/qa/tf/ci/smoketest/provider.tf b/qa/tf/ci/smoketest/provider.tf new file mode 100644 index 000000000..c0fc95d9d --- /dev/null +++ b/qa/tf/ci/smoketest/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.region + profile = var.profile +} \ No newline at end of file diff --git a/qa/tf/ci/smoketest/terraform.tfstate.backup b/qa/tf/ci/smoketest/terraform.tfstate.backup new file mode 100644 index 000000000..f2f74f7a8 --- /dev/null +++ b/qa/tf/ci/smoketest/terraform.tfstate.backup @@ -0,0 +1,1596 @@ +{ + "version": 4, + "terraform_version": "1.1.2", + "serial": 255, + "lineage": "bf91272c-d504-5c98-ce46-2ced1888bf74", + "outputs": { + "data_node_ips": { + "value": [ + "10.0.1.152" + ], + "type": [ + "tuple", + [ + "string" + ] + ] + }, + "ingest_ips": { + "value": [ + "3.128.203.136" + ], + "type": [ + "tuple", + [ + "string" + ] + ] + } + }, + "resources": [ + { + "module": "module.ci-cluster", + "mode": "data", + "type": "aws_ami", + "name": "amazon_linux_2", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architecture": "arm64", + "arn": "arn:aws:ec2:us-east-2::image/ami-0b09f36be67d32fff", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": { + "delete_on_termination": "true", + "encrypted": "false", + "iops": "0", + "snapshot_id": "snap-0617b00e90bae012b", + "throughput": "0", + "volume_size": "8", + "volume_type": "gp2" + }, + "no_device": "", + "virtual_name": "" + } + ], + "creation_date": "2021-12-01T19:36:11.000Z", + "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211201.0 arm64 HVM gp2", + "ena_support": true, + "executable_users": null, + "filter": [ + { + "name": "architecture", + "values": [ + "arm64" + ] + }, + { + "name": "name", + "values": [ + "amzn2-ami-hvm-*" + ] + }, + { + "name": "virtualization-type", + "values": [ + "hvm" + ] + } + ], + "hypervisor": "xen", + "id": "ami-0b09f36be67d32fff", + "image_id": "ami-0b09f36be67d32fff", + "image_location": "amazon/amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "image_owner_alias": "amazon", + "image_type": "machine", + "kernel_id": null, + "most_recent": true, + "name": "amzn2-ami-hvm-2.0.20211201.0-arm64-gp2", + "name_regex": null, + "owner_id": "137112412989", + "owners": [ + "amazon" + ], + "platform": null, + "platform_details": "Linux/UNIX", + "product_codes": [], + "public": true, + "ramdisk_id": null, + "root_device_name": "/dev/xvda", + "root_device_type": "ebs", + "root_snapshot_id": "snap-0617b00e90bae012b", + "sriov_net_support": "simple", + "state": "available", + "state_reason": { + "code": "UNSET", + "message": "UNSET" + }, + "tags": {}, + "usage_operation": "RunInstances", + "virtualization_type": "hvm" + }, + "sensitive_attributes": [] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_iam_instance_profile", + "name": "fb_cluster_node_profile", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::941206295814:instance-profile/fb_cluster_node_profile", + "create_date": "2022-01-05T19:35:38Z", + "id": "fb_cluster_node_profile", + "name": "fb_cluster_node_profile", + "name_prefix": null, + "path": "/", + "role": "fb_cluster_node", + "tags": null, + "tags_all": {}, + "unique_id": "AIPA5WJCEKUDB6OWYZPTW" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.aws_iam_role.fb_cluster_node_role" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_iam_role", + "name": "fb_cluster_node_role", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::941206295814:role/fb_cluster_node", + "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", + "create_date": "2022-01-05T19:35:35Z", + "description": "", + "force_detach_policies": false, + "id": "fb_cluster_node", + "inline_policy": [ + { + "name": "ec2_read_all", + "policy": "{\"Statement\":[{\"Action\":[\"ec2:Describe*\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}" + } + ], + "managed_policy_arns": [], + "max_session_duration": 3600, + "name": "fb_cluster_node", + "name_prefix": "", + "path": "/", + "permissions_boundary": null, + "tags": null, + "tags_all": {}, + "unique_id": "AROA5WJCEKUDN4ZISIR3N" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_instance", + "name": "fb_cluster_nodes", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "ami": "ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2:941206295814:instance/i-08ac53ed9d7ac2ac1", + "associate_public_ip_address": false, + "availability_zone": "us-east-2a", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 2, + "cpu_threads_per_core": 1, + "credit_specification": [], + "disable_api_termination": false, + "ebs_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/sdb", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "snapshot_id": "", + "tags": {}, + "throughput": 125, + "volume_id": "vol-0041b5d58bde3da13", + "volume_size": 100, + "volume_type": "gp3" + } + ], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "fb_cluster_node_profile", + "id": "i-08ac53ed9d7ac2ac1", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "m6g.large", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "gitlab-featurebase-ci", + "launch_template": [], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional" + } + ], + "monitoring": true, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-0b00c3636cad95ae5", + "private_dns": "ip-10-0-1-152.us-east-2.compute.internal", + "private_ip": "10.0.1.152", + "public_dns": "", + "public_ip": "", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "tags": null, + "throughput": 125, + "volume_id": "vol-0f12ff15c510db547", + "volume_size": 20, + "volume_type": "gp3" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-0dcb2339122bafc95", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV-featurebase-cluster-0", + "Prefix": "smoke-X8A0attf2zz6EhnV", + "Role": "cluster_node" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV-featurebase-cluster-0", + "Prefix": "smoke-X8A0attf2zz6EhnV", + "Role": "cluster_node" + }, + "tenancy": "default", + "timeouts": null, + "user_data": "efedaaed014dc69321f5e633ebc5dc436baf7b1e", + "user_data_base64": null, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-0580fa9cbc0493d77" + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.ci-cluster.aws_iam_role.fb_cluster_node_role", + "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", + "module.ci-cluster.aws_security_group.featurebase", + "module.ci-cluster.data.aws_ami.amazon_linux_2", + "module.ci-cluster.module.vpc.aws_subnet.private", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_instance", + "name": "fb_ingest", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "ami": "ami-0b09f36be67d32fff", + "arn": "arn:aws:ec2:us-east-2:941206295814:instance/i-07f30af63526e9a6b", + "associate_public_ip_address": true, + "availability_zone": "us-east-2a", + "capacity_reservation_specification": [ + { + "capacity_reservation_preference": "open", + "capacity_reservation_target": [] + } + ], + "cpu_core_count": 8, + "cpu_threads_per_core": 1, + "credit_specification": [], + "disable_api_termination": false, + "ebs_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/sdb", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "snapshot_id": "", + "tags": {}, + "throughput": 125, + "volume_id": "vol-0a7270e5a68789fbb", + "volume_size": 100, + "volume_type": "gp3" + } + ], + "ebs_optimized": false, + "enclave_options": [ + { + "enabled": false + } + ], + "ephemeral_block_device": [], + "get_password_data": false, + "hibernation": false, + "host_id": null, + "iam_instance_profile": "fb_cluster_node_profile", + "id": "i-07f30af63526e9a6b", + "instance_initiated_shutdown_behavior": "stop", + "instance_state": "running", + "instance_type": "c6g.2xlarge", + "ipv6_address_count": 0, + "ipv6_addresses": [], + "key_name": "gitlab-featurebase-ci", + "launch_template": [], + "metadata_options": [ + { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional" + } + ], + "monitoring": true, + "network_interface": [], + "outpost_arn": "", + "password_data": "", + "placement_group": "", + "placement_partition_number": null, + "primary_network_interface_id": "eni-0c855451ce29250c9", + "private_dns": "ip-10-0-101-214.us-east-2.compute.internal", + "private_ip": "10.0.101.214", + "public_dns": "", + "public_ip": "3.128.203.136", + "root_block_device": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "encrypted": false, + "iops": 3000, + "kms_key_id": "", + "tags": null, + "throughput": 125, + "volume_id": "vol-021c14b5593bdd999", + "volume_size": 20, + "volume_type": "gp3" + } + ], + "secondary_private_ips": [], + "security_groups": [], + "source_dest_check": true, + "subnet_id": "subnet-09f6ef1380254341b", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV-featurebase-ingest-0", + "Prefix": "smoke-X8A0attf2zz6EhnV", + "Role": "ingest_node" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV-featurebase-ingest-0", + "Prefix": "smoke-X8A0attf2zz6EhnV", + "Role": "ingest_node" + }, + "tenancy": "default", + "timeouts": null, + "user_data": "38da76e780fedb19ef551fe9d3c84540d1823453", + "user_data_base64": null, + "volume_tags": null, + "vpc_security_group_ids": [ + "sg-08089f99e02a87452" + ] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", + "dependencies": [ + "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", + "module.ci-cluster.aws_iam_role.fb_cluster_node_role", + "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", + "module.ci-cluster.aws_security_group.ingest", + "module.ci-cluster.data.aws_ami.amazon_linux_2", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_key_pair", + "name": "gitlab-featurebase-ci", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:key-pair/gitlab-featurebase-ci", + "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69", + "id": "gitlab-featurebase-ci", + "key_name": "gitlab-featurebase-ci", + "key_name_prefix": "", + "key_pair_id": "key-0a36382ce2a87c44e", + "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_security_group", + "name": "featurebase", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:security-group/sg-0580fa9cbc0493d77", + "description": "Allow featurebase inbound traffic", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-0580fa9cbc0493d77", + "ingress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "SSH", + "from_port": 22, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 22 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "GRPC from Internal", + "from_port": 20101, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 20101 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "PostgreSQL from Internal", + "from_port": 55432, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 55432 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "TLS from Internal", + "from_port": 10101, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10101 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "etcd from internal 2", + "from_port": 10401, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10401 + }, + { + "cidr_blocks": [ + "10.0.0.0/16" + ], + "description": "etcd from internal", + "from_port": 10301, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10301 + } + ], + "name": "allow_featurebase", + "name_prefix": "", + "owner_id": "941206295814", + "revoke_rules_on_delete": false, + "tags": { + "Name": "allow_featurebase" + }, + "tags_all": { + "Name": "allow_featurebase" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this" + ] + } + ] + }, + { + "module": "module.ci-cluster", + "mode": "managed", + "type": "aws_security_group", + "name": "ingest", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:security-group/sg-08089f99e02a87452", + "description": "Allow ingest inbound traffic", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-08089f99e02a87452", + "ingress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 10101, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 10101 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "SSH", + "from_port": 22, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 22 + } + ], + "name": "allow_ingest", + "name_prefix": "", + "owner_id": "941206295814", + "revoke_rules_on_delete": false, + "tags": { + "Name": "allow_ingest" + }, + "tags_all": { + "Name": "allow_ingest" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_eip", + "name": "nat", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "address": null, + "allocation_id": "eipalloc-099cedbe3c5820bbe", + "associate_with_private_ip": null, + "association_id": "", + "carrier_ip": "", + "customer_owned_ip": "", + "customer_owned_ipv4_pool": "", + "domain": "vpc", + "id": "eipalloc-099cedbe3c5820bbe", + "instance": "", + "network_border_group": "us-east-2", + "network_interface": "", + "private_dns": null, + "private_ip": "", + "public_dns": "ec2-3-17-85-138.us-east-2.compute.amazonaws.com", + "public_ip": "3.17.85.138", + "public_ipv4_pool": "amazon", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxODAwMDAwMDAwMDAsInJlYWQiOjkwMDAwMDAwMDAwMCwidXBkYXRlIjozMDAwMDAwMDAwMDB9fQ==" + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "address": null, + "allocation_id": "eipalloc-0fca65223cdfd313a", + "associate_with_private_ip": null, + "association_id": "", + "carrier_ip": "", + "customer_owned_ip": "", + "customer_owned_ipv4_pool": "", + "domain": "vpc", + "id": "eipalloc-0fca65223cdfd313a", + "instance": "", + "network_border_group": "us-east-2", + "network_interface": "", + "private_dns": null, + "private_ip": "", + "public_dns": "ec2-3-135-112-2.us-east-2.compute.amazonaws.com", + "public_ip": "3.135.112.2", + "public_ipv4_pool": "amazon", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxODAwMDAwMDAwMDAsInJlYWQiOjkwMDAwMDAwMDAwMCwidXBkYXRlIjozMDAwMDAwMDAwMDB9fQ==" + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "address": null, + "allocation_id": "eipalloc-09323e2a47e394df7", + "associate_with_private_ip": null, + "association_id": "", + "carrier_ip": "", + "customer_owned_ip": "", + "customer_owned_ipv4_pool": "", + "domain": "vpc", + "id": "eipalloc-09323e2a47e394df7", + "instance": "", + "network_border_group": "us-east-2", + "network_interface": "", + "private_dns": null, + "private_ip": "", + "public_dns": "ec2-3-128-36-233.us-east-2.compute.amazonaws.com", + "public_ip": "3.128.36.233", + "public_ipv4_pool": "amazon", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc": true + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjoxODAwMDAwMDAwMDAsInJlYWQiOjkwMDAwMDAwMDAwMCwidXBkYXRlIjozMDAwMDAwMDAwMDB9fQ==" + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_internet_gateway", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:internet-gateway/igw-069e52ef204598fdd", + "id": "igw-069e52ef204598fdd", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_nat_gateway", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "allocation_id": "eipalloc-099cedbe3c5820bbe", + "connectivity_type": "public", + "id": "nat-01960cd93ff94dc13", + "network_interface_id": "eni-0da9e0562c5e62a35", + "private_ip": "10.0.101.138", + "public_ip": "3.17.85.138", + "subnet_id": "subnet-09f6ef1380254341b", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "allocation_id": "eipalloc-0fca65223cdfd313a", + "connectivity_type": "public", + "id": "nat-03883494dfdfcc234", + "network_interface_id": "eni-0f234f97b35663c64", + "private_ip": "10.0.102.238", + "public_ip": "3.135.112.2", + "subnet_id": "subnet-063459780d5b84c86", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "allocation_id": "eipalloc-09323e2a47e394df7", + "connectivity_type": "public", + "id": "nat-05bbd0bcd20938896", + "network_interface_id": "eni-0a36a85d8383cac8e", + "private_ip": "10.0.103.228", + "public_ip": "3.128.36.233", + "subnet_id": "subnet-0eae785053155c08b", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "private_nat_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "", + "id": "r-rtb-06122e232dba3e71d1080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "nat-01960cd93ff94dc13", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-06122e232dba3e71d", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_nat_gateway.this", + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "", + "id": "r-rtb-0757c96161330927f1080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "nat-03883494dfdfcc234", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-0757c96161330927f", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_nat_gateway.this", + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "", + "id": "r-rtb-0584788728d6485171080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "nat-05bbd0bcd20938896", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-0584788728d648517", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_eip.nat", + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_nat_gateway.this", + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route", + "name": "public_internet_gateway", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "carrier_gateway_id": "", + "destination_cidr_block": "0.0.0.0/0", + "destination_ipv6_cidr_block": "", + "destination_prefix_list_id": "", + "egress_only_gateway_id": "", + "gateway_id": "igw-069e52ef204598fdd", + "id": "r-rtb-08768a10ff241df961080289494", + "instance_id": "", + "instance_owner_id": "", + "local_gateway_id": "", + "nat_gateway_id": "", + "network_interface_id": "", + "origin": "CreateRoute", + "route_table_id": "rtb-08768a10ff241df96", + "state": "active", + "timeouts": { + "create": "5m", + "delete": null, + "update": null + }, + "transit_gateway_id": "", + "vpc_endpoint_id": "", + "vpc_peering_connection_id": "" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_internet_gateway.this", + "module.ci-cluster.module.vpc.aws_route_table.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:route-table/rtb-06122e232dba3e71d", + "id": "rtb-06122e232dba3e71d", + "owner_id": "941206295814", + "propagating_vgws": [], + "route": [], + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:route-table/rtb-0757c96161330927f", + "id": "rtb-0757c96161330927f", + "owner_id": "941206295814", + "propagating_vgws": [], + "route": [], + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:route-table/rtb-0584788728d648517", + "id": "rtb-0584788728d648517", + "owner_id": "941206295814", + "propagating_vgws": [], + "route": [], + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route_table", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:route-table/rtb-08768a10ff241df96", + "id": "rtb-08768a10ff241df96", + "owner_id": "941206295814", + "propagating_vgws": [], + "route": [], + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6MzAwMDAwMDAwMDAwLCJ1cGRhdGUiOjEyMDAwMDAwMDAwMH19", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0e02368a84fbc483d", + "route_table_id": "rtb-06122e232dba3e71d", + "subnet_id": "subnet-0dcb2339122bafc95" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.private", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-03a94dc33d8ce77ee", + "route_table_id": "rtb-0757c96161330927f", + "subnet_id": "subnet-0981f7f434ca09e8e" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.private", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0bc6944b6e56191de", + "route_table_id": "rtb-0584788728d648517", + "subnet_id": "subnet-0353db6d8c1c7ad43" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.private", + "module.ci-cluster.module.vpc.aws_subnet.private", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_route_table_association", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-03dfd5b74f16f8cd8", + "route_table_id": "rtb-08768a10ff241df96", + "subnet_id": "subnet-09f6ef1380254341b" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.public", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0628c1dc5f82384f6", + "route_table_id": "rtb-08768a10ff241df96", + "subnet_id": "subnet-063459780d5b84c86" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.public", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 0, + "attributes": { + "gateway_id": "", + "id": "rtbassoc-0b730b5a81f9e2de4", + "route_table_id": "rtb-08768a10ff241df96", + "subnet_id": "subnet-0eae785053155c08b" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_route_table.public", + "module.ci-cluster.module.vpc.aws_subnet.public", + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "private", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-0dcb2339122bafc95", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2a", + "availability_zone_id": "use2-az1", + "cidr_block": "10.0.1.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-0dcb2339122bafc95", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-0981f7f434ca09e8e", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2b", + "availability_zone_id": "use2-az2", + "cidr_block": "10.0.2.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-0981f7f434ca09e8e", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-0353db6d8c1c7ad43", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2c", + "availability_zone_id": "use2-az3", + "cidr_block": "10.0.3.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-0353db6d8c1c7ad43", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": false, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_subnet", + "name": "public", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-09f6ef1380254341b", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2a", + "availability_zone_id": "use2-az1", + "cidr_block": "10.0.101.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-09f6ef1380254341b", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": true, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 1, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-063459780d5b84c86", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2b", + "availability_zone_id": "use2-az2", + "cidr_block": "10.0.102.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-063459780d5b84c86", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": true, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + }, + { + "index_key": 2, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:subnet/subnet-0eae785053155c08b", + "assign_ipv6_address_on_creation": false, + "availability_zone": "us-east-2c", + "availability_zone_id": "use2-az3", + "cidr_block": "10.0.103.0/24", + "customer_owned_ipv4_pool": "", + "id": "subnet-0eae785053155c08b", + "ipv6_cidr_block": "", + "ipv6_cidr_block_association_id": "", + "map_customer_owned_ip_on_launch": false, + "map_public_ip_on_launch": true, + "outpost_arn": "", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "timeouts": null, + "vpc_id": "vpc-035fcf75548026b23" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "module.ci-cluster.module.vpc.aws_vpc.this", + "module.ci-cluster.module.vpc.aws_vpc_ipv4_cidr_block_association.this" + ] + } + ] + }, + { + "module": "module.ci-cluster.module.vpc", + "mode": "managed", + "type": "aws_vpc", + "name": "this", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:941206295814:vpc/vpc-035fcf75548026b23", + "assign_generated_ipv6_cidr_block": false, + "cidr_block": "10.0.0.0/16", + "default_network_acl_id": "acl-0db479d99cf0759f8", + "default_route_table_id": "rtb-0a264d3195bef2a21", + "default_security_group_id": "sg-025b70c95323fd7e6", + "dhcp_options_id": "dopt-0398725faf4f782c8", + "enable_classiclink": null, + "enable_classiclink_dns_support": null, + "enable_dns_hostnames": false, + "enable_dns_support": true, + "id": "vpc-035fcf75548026b23", + "instance_tenancy": "default", + "ipv4_ipam_pool_id": null, + "ipv4_netmask_length": null, + "ipv6_association_id": "", + "ipv6_cidr_block": "", + "ipv6_ipam_pool_id": null, + "ipv6_netmask_length": null, + "main_route_table_id": "rtb-0a264d3195bef2a21", + "owner_id": "941206295814", + "tags": { + "Name": "smoke-X8A0attf2zz6EhnV" + }, + "tags_all": { + "Name": "smoke-X8A0attf2zz6EhnV" + } + }, + "sensitive_attributes": [], + "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" + } + ] + } + ] +} diff --git a/qa/tf/ci/smoketest/tf.auto.tfvars b/qa/tf/ci/smoketest/tf.auto.tfvars new file mode 100644 index 000000000..ac6de62a6 --- /dev/null +++ b/qa/tf/ci/smoketest/tf.auto.tfvars @@ -0,0 +1,2 @@ +region = "us-east-2" +profile = "service-terraform" \ No newline at end of file diff --git a/qa/tf/ci/smoketest/variables.tf b/qa/tf/ci/smoketest/variables.tf new file mode 100644 index 000000000..bab877497 --- /dev/null +++ b/qa/tf/ci/smoketest/variables.tf @@ -0,0 +1,24 @@ +variable "region" { + description = "The AWS region in which the VPC should be built" + type = string +} + +variable "profile" { + description = "The name of the AWS profile Terraform should use for auth." + type = string +} + +variable "gitlab_token" { + description = "The API token for taking to Gitlab API - expected to come from an env variable." + type = string +} + +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} + +variable "branch" { + type = string + description = "The branch we are on" +} \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/README.md b/qa/tf/gauntlet/samsung/README.md new file mode 100644 index 000000000..e7b633516 --- /dev/null +++ b/qa/tf/gauntlet/samsung/README.md @@ -0,0 +1,35 @@ +With terraform installed (`brew install terraform` if not)... + +You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down. + +## Other prerequisites: +Please read these carefully. + +Be in the `tf` directory (e.g., when you try to run a `terraform` command, the output of `pwd` should be `.../featurebase/qa/tf`) + +Currently, the path to the terraform module is using a local reference, i.e., in `main.tf`, the source line is assuming that you have `molecula-terraform` project installed locally, such that the `molecular-terraform` project and `featurebase` have the same parent directory (e.g., `...A/featurebase/qa/tf` and `...A/molecular-terraform/aws/.modules/featurebase-cluster` should both be valid paths). + +In addition, you must currently have a local copy of the `fb901` branch for the `molecular-terraform` project (located in the previously specified directory). + +Last thing, there is a key that is currently in 1Password (in the `Shared` vault, called `gitlab-featurebase-ci AWS key`) that must be in `~/.ssh/`, `chmod 400`, named `gitlab-featurebase-ci.pem`. You need this key to SSH to these instances. Assuming an `~/.ssh/config` like the following (append to the top of yours) +``` +Host test_* + User ec2-user + IdentityFile ~/.ssh/gitlab-featurebase-ci.pem +Host test_ingest + HostName 3.143.237.165 +Host test_node + HostName 10.0.1.142 + ProxyJump test_ingest +``` +except with the `test_ingest`'s `HostName` being the public, `ingest_ips` output from `terraform output` and `test_node`'s `HostName` being one of the private, `data_node_ips` output from `terraform output`. (Hopefully the rationale to use the ssh config to do the jumping like this makes sense; you can do `ssh test_ingest` or `ssh test_node` with minimal further fiddling.) + +OR specify cert to us directly thus: + +`ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@ip_address` + +-A is used to ensure key forwarding. + +### TODOs +* We need a `user-data.sh` script which sets up/installs featurebase (possibly installs go, most likely pulls the artifacts from GitLab; sets up featurebase on both the node and data workers). +* Logs get sent to DataDog? diff --git a/qa/tf/gauntlet/samsung/main.tf b/qa/tf/gauntlet/samsung/main.tf new file mode 100644 index 000000000..8a5101d7c --- /dev/null +++ b/qa/tf/gauntlet/samsung/main.tf @@ -0,0 +1,14 @@ +module "samsung-cluster" { + source = "../../.modules/featurebase-cluster" + cluster_prefix = "samsung-gauntlet" + region = var.region + profile = var.profile + fb_data_node_type = "m6g.xlarge" + fb_data_disk_iops = 10000 + fb_data_node_count = 3 + fb_ingest_type = "m6g.large" + fb_ingest_disk_iops = 10000 + fb_ingest_node_count = 1 + gitlab_token = var.gitlab_token + branch = var.branch +} diff --git a/qa/tf/gauntlet/samsung/outputs.tf b/qa/tf/gauntlet/samsung/outputs.tf new file mode 100644 index 000000000..0c405bed0 --- /dev/null +++ b/qa/tf/gauntlet/samsung/outputs.tf @@ -0,0 +1,9 @@ +output "ingest_ips" { + description = "List of ingest IPs" + value = module.samsung-cluster.ingest_ips +} + +output "data_node_ips" { + description = "List of data node IPs" + value = module.samsung-cluster.data_node_ips +} \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/provider.tf b/qa/tf/gauntlet/samsung/provider.tf new file mode 100644 index 000000000..c0fc95d9d --- /dev/null +++ b/qa/tf/gauntlet/samsung/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.region + profile = var.profile +} \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/tf.auto.tfvars b/qa/tf/gauntlet/samsung/tf.auto.tfvars new file mode 100644 index 000000000..ac6de62a6 --- /dev/null +++ b/qa/tf/gauntlet/samsung/tf.auto.tfvars @@ -0,0 +1,2 @@ +region = "us-east-2" +profile = "service-terraform" \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/variables.tf b/qa/tf/gauntlet/samsung/variables.tf new file mode 100644 index 000000000..bab877497 --- /dev/null +++ b/qa/tf/gauntlet/samsung/variables.tf @@ -0,0 +1,24 @@ +variable "region" { + description = "The AWS region in which the VPC should be built" + type = string +} + +variable "profile" { + description = "The name of the AWS profile Terraform should use for auth." + type = string +} + +variable "gitlab_token" { + description = "The API token for taking to Gitlab API - expected to come from an env variable." + type = string +} + +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} + +variable "branch" { + type = string + description = "The branch we are on" +} \ No newline at end of file diff --git a/rbf/cursor.go b/rbf/cursor.go index eda6bec1d..2419bc5f8 100644 --- a/rbf/cursor.go +++ b/rbf/cursor.go @@ -932,6 +932,10 @@ func (c *Cursor) First() error { case PageTypeBranch: elem.index = 0 + if n := readCellN(buf); elem.index >= n { // branch cell index must less than cell count + return fmt.Errorf("branch cell index out of range: pgno=%d i=%d n=%d", elem.pgno, elem.index, n) + } + // Read cell pgno into the next stack level. cell := readBranchCell(buf, elem.index) diff --git a/rbf/db.go b/rbf/db.go index 65dd4fbea..6e4955f94 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -7,6 +7,8 @@ import ( "io" "os" "path/filepath" + "runtime/debug" + "sort" "sync" "syscall" @@ -637,6 +639,7 @@ func (db *DB) Begin(writable bool) (_ *Tx, err error) { pageMap: db.pageMap, walPageN: db.walPageN, writable: writable, + stack: debug.Stack(), // DEBUG DeleteEmptyContainer: true, } @@ -815,6 +818,20 @@ func (db *DB) getCursor(tx *Tx) *Cursor { return c } +func (db *DB) DebugInfo() *DebugInfo { + info := &DebugInfo{Path: db.Path} + for tx := range db.txs { + info.Txs = append(info.Txs, tx.DebugInfo()) + } + sort.Slice(info.Txs, func(i, j int) bool { return info.Txs[i].Ptr < info.Txs[j].Ptr }) + return info +} + +type DebugInfo struct { + Path string `json:"path"` + Txs []*TxDebugInfo `json:"txs"` +} + // Shared pool for in-memory database pages. // These are used before being flushed to disk. var pagePool = &sync.Pool{ diff --git a/rbf/db_test.go b/rbf/db_test.go index c0eb3b3c7..8bae550bb 100644 --- a/rbf/db_test.go +++ b/rbf/db_test.go @@ -339,6 +339,21 @@ func TestDB_MultiTx(t *testing.T) { } } +func TestDB_DebugInfo(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + tx := MustBegin(t, db, true) + defer tx.Rollback() + + info := db.DebugInfo() + if got, want := info.Path, db.Path; got != want { + t.Fatalf("Path=%q, want %q", got, want) + } else if got, want := len(info.Txs), 1; got != want { + t.Fatalf("len(Txs)=%d, want %d", got, want) + } +} + // premake pool of random values const randPool = (1 << 18) diff --git a/rbf/rbf.go b/rbf/rbf.go index 74a5135eb..4a66ba309 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -799,3 +799,46 @@ func (m *Metric) Inc(d time.Duration) { fmt.Printf("metric:%10s avg=%dns\n", m.name, int(m.d)/m.n) } } + +// ErrorList represents a list of errors. +type ErrorList []error + +// Err returns the list if it contains errors. Otherwise returns nil. +func (a ErrorList) Err() error { + if len(a) > 0 { + return a + } + return nil +} + +func (a ErrorList) Error() string { + switch len(a) { + case 0: + return "no errors" + case 1: + return a[0].Error() + } + return fmt.Sprintf("%s (and %d more errors)", a[0], len(a)-1) +} + +func (a ErrorList) FullError() string { + if len(a) == 0 { + return "" + } + + var buf bytes.Buffer + for _, err := range a { + fmt.Fprintln(&buf, err) + } + return buf.String() +} + +// Append appends an error to the list. If err is an ErrorList then all errors are appended. +func (a *ErrorList) Append(err error) { + switch err := err.(type) { + case ErrorList: + *a = append(*a, err...) + default: + *a = append(*a, err) + } +} diff --git a/rbf/rbf/testdata/check/bad-freelist/data b/rbf/rbf/testdata/check/bad-freelist/data new file mode 100644 index 000000000..8b03c7b02 Binary files /dev/null and b/rbf/rbf/testdata/check/bad-freelist/data differ diff --git a/rbf/rbf/testdata/check/bad-freelist/wal b/rbf/rbf/testdata/check/bad-freelist/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go index 4071a1a95..17605a6fc 100644 --- a/rbf/rbf_test.go +++ b/rbf/rbf_test.go @@ -54,17 +54,30 @@ func NewDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { if err != nil { panic(err) } + return NewDBAt(tb, path, cfg...) +} +// NewDBAt returns a new instance of DB with a given path. +func NewDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB { var cfg0 *rbfcfg.Config if len(cfg) > 0 { cfg0 = cfg[0] } - db := rbf.NewDB(path, cfg0) - return db + return rbf.NewDB(path, cfg0) } // MustOpenDB returns a db opened on a temporary file. On error, fail test. func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { + tb.Helper() + path, err := testhook.TempDir(tb, "rbfdb") + if err != nil { + panic(err) + } + return MustOpenDBAt(tb, path, cfg...) +} + +// MustOpenDBAt returns a db opened on an existing file. On error, fail test. +func MustOpenDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB { tb.Helper() if len(cfg) == 0 || cfg[0] == nil { newconf := rbfcfg.NewDefaultConfig() @@ -73,7 +86,7 @@ func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB { } else if cfg[0].Logger == nil { cfg[0].Logger = logger.NewLogfLogger(tb) } - db := NewDB(tb, cfg...) + db := NewDBAt(tb, path, cfg...) if err := db.Open(); err != nil { tb.Fatal(err) } diff --git a/rbf/testdata/check/bad-bitmap/data b/rbf/testdata/check/bad-bitmap/data new file mode 100644 index 000000000..6cc32c0a1 Binary files /dev/null and b/rbf/testdata/check/bad-bitmap/data differ diff --git a/rbf/testdata/check/bad-bitmap/wal b/rbf/testdata/check/bad-bitmap/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/testdata/check/bad-freelist/data b/rbf/testdata/check/bad-freelist/data new file mode 100644 index 000000000..a762ee9db Binary files /dev/null and b/rbf/testdata/check/bad-freelist/data differ diff --git a/rbf/testdata/check/bad-freelist/wal b/rbf/testdata/check/bad-freelist/wal new file mode 100644 index 000000000..e69de29bb diff --git a/rbf/tx.go b/rbf/tx.go index 5c6c7f7c6..c45380469 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -65,6 +65,9 @@ type Tx struct { // manages to trigger a *deallocation* (which I don't think should be // happening), we'll process that one after the current list is processed. pendingFreelistAdds []uint32 + + // DEBUG + stack []byte } func (tx *Tx) DBPath() string { @@ -735,10 +738,11 @@ func (tx *Tx) Check() error { return ErrTxClosed } + var errorList ErrorList if err := tx.checkPageAllocations(); err != nil { - return fmt.Errorf("page allocations: %w", err) + errorList.Append(err) } - return nil + return errorList.Err() } func (tx *Tx) checkPage(pgno, parent, typ uint32) error { @@ -764,14 +768,15 @@ func (tx *Tx) checkBranchPage(pgno, parent, typ uint32) error { // checkPageAllocations ensures that all pages are either in-use or on the freelist. func (tx *Tx) checkPageAllocations() error { + var errorList ErrorList freePageSet, err := tx.freePageSet() if err != nil { - return err + errorList.Append(err) } inusePageSet, err := tx.inusePageSet() if err != nil { - return err + errorList.Append(err) } // Iterate over all pages and ensure they are either in-use or free. @@ -782,26 +787,23 @@ func (tx *Tx) checkPageAllocations() error { _, isFree := freePageSet[pgno] if isInuse && isFree { - return fmt.Errorf("page in-use & free: pgno=%d", pgno) - } else if !isInuse && !isFree { - page, _, err := tx.readPage(pgno) - if err != nil { - return err - } - flags := readFlags(page) - if flags == PageTypeBranch || flags == PageTypeLeaf { - return fmt.Errorf("page not in-use & not free: pgno=%d", pgno) - } - //assuming its a bitmap so its ok TODO ben? - return nil + errorList.Append(fmt.Errorf("page in-use & free: pgno=%d", pgno)) + continue + } + + if !isInuse && !isFree { + errorList.Append(fmt.Errorf("page not in-use & not free: pgno=%d", pgno)) + continue } } - return nil + return errorList.Err() } // freePageSet returns the set of pages in the freelist. func (tx *Tx) freePageSet() (map[uint32]struct{}, error) { + var errorList ErrorList + m := make(map[uint32]struct{}) c := Cursor{tx: tx} c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} @@ -813,18 +815,20 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) { for { if err := c.Next(); err == io.EOF { - return m, nil + return m, errorList.Err() } else if err != nil { - return m, err + errorList.Append(err) + return m, errorList.Err() } elem := &c.stack.elems[c.stack.top] leafPage, _, err := c.tx.readPage(elem.pgno) if err != nil { - return nil, err + errorList.Append(fmt.Errorf("cannot read free page: pgno=%d err=%w", elem.pgno, err)) + continue } - cell := readLeafCell(leafPage, elem.index) + cell := readLeafCell(leafPage, elem.index) for _, v := range cell.Values(tx) { pgno := uint32((cell.Key << 16) | uint64(v)) m[pgno] = struct{}{} @@ -834,6 +838,7 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) { // inusePageSet returns the set of pages in use by the root records or b-trees. func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { + var errorList ErrorList m := make(map[uint32]struct{}) m[0] = struct{}{} // meta page @@ -843,15 +848,24 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { page, _, err := tx.readPage(pgno) if err != nil { - return nil, err + errorList.Append(err) + break } pgno = WalkRootRecordPages(page) } // Traverse freelist and mark pages as in-use. - if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32) error { + if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32, err error) error { + if err != nil { + errorList.Append(err) + return nil + } + m[pgno] = struct{}{} - return tx.checkPage(pgno, parent, typ) + if err := tx.checkPage(pgno, parent, typ); err != nil { + errorList.Append(err) + } + return nil }); err != nil { return m, err } @@ -859,22 +873,28 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) { // Traverse every b-tree and mark pages as in-use. records, err := tx.RootRecords() if err != nil { - return m, err - } + errorList.Append(err) + } else { + for itr := records.Iterator(); !itr.Done(); { + _, pgno := itr.Next() - for itr := records.Iterator(); !itr.Done(); { - _, pgno := itr.Next() + if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error { + if err != nil { + errorList.Append(err) + } - if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error { - m[pgno] = struct{}{} - - return tx.checkPage(pgno, parent, typ) - }); err != nil { - return m, err + m[pgno] = struct{}{} + if err := tx.checkPage(pgno, parent, typ); err != nil { + errorList.Append(err) + } + return nil + }); err != nil { + return m, err + } } } - return m, nil + return m, errorList.Err() } // GetSizeBytesWithPrefix returns the size of bitmaps with a given key prefix. @@ -894,9 +914,9 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) { } // Traverse the bitmap's b-tree and count the bytes for each page. - if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error { + if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error { n += PageSize - return nil + return err }); err != nil { return 0, err } @@ -905,21 +925,19 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) { } // walkTree recursively iterates over a page and all its children. -func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) error) error { +func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32, err error) error) error { // Read page and iterate over children. page, _, err := tx.readPage(pgno) if err != nil { - return err + return fn(pgno, parent, 0, fmt.Errorf("cannot read page: pgno=%d parent=%d err=%s", pgno, parent, err)) } - // Execute callback. - typ := readFlags(page) - if err := fn(pgno, parent, typ); err != nil { - return err - } - - switch typ { + switch typ := readFlags(page); typ { case PageTypeBranch: + if err := fn(pgno, parent, typ, nil); err != nil { + return err + } + for i, n := 0, readCellN(page); i < n; i++ { cell := readBranchCell(page, i) if err := tx.walkTree(cell.ChildPgno, pgno, fn); err != nil { @@ -927,18 +945,24 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er } } return nil + case PageTypeLeaf: + if err := fn(pgno, parent, typ, nil); err != nil { + return err + } + // Execute callback only for bitmap pages pointed to by this leaf. for i, n := 0, readCellN(page); i < n; i++ { if cell := readLeafCell(page, i); cell.Type == ContainerTypeBitmapPtr { - if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap); err != nil { + if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap, nil); err != nil { return err } } } return nil + default: - return fmt.Errorf("rbf.Tx.forEachTreePage(): invalid page type: pgno=%d type=%d", pgno, typ) + return fn(pgno, parent, typ, fmt.Errorf("invalid page type: pgno=%d parent=%d type=%d", pgno, parent, typ)) } } @@ -1903,6 +1927,7 @@ func (tx *Tx) Pages(pgnos []uint32) ([]Page, error) { // PageInfos returns meta data about all pages in the database. func (tx *Tx) PageInfos() ([]PageInfo, error) { + var errorList ErrorList infos := make([]PageInfo, tx.PageN()) // Read meta page info. @@ -1916,7 +1941,8 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) { for pgno := metaInfo.RootRecordPageNo; pgno != 0; { info, err := tx.rootRecordPageInfo(pgno) if err != nil { - return nil, err + errorList.Append(err) + break } infos[pgno] = info pgno = info.Next @@ -1924,33 +1950,34 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) { // Traverse freelist and mark pages as in-use. if err := tx.walkPageInfo(infos, metaInfo.FreelistPageNo, "freelist"); err != nil { - return nil, err + errorList.Append(err) } // Traverse every b-tree and mark pages as in-use. records, err := tx.RootRecords() if err != nil { - return nil, err - } + errorList.Append(err) + } else { + for itr := records.Iterator(); !itr.Done(); { + name, pgno := itr.Next() - for itr := records.Iterator(); !itr.Done(); { - name, pgno := itr.Next() - - if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil { - return nil, err + if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil { + errorList.Append(err) + } } } // Build page info objects for each free page. freePageSet, err := tx.freePageSet() if err != nil { - return nil, err - } - for pgno := range freePageSet { - infos[pgno] = &FreePageInfo{Pgno: pgno} + errorList.Append(err) + } else { + for pgno := range freePageSet { + infos[pgno] = &FreePageInfo{Pgno: pgno} + } } - return infos, nil + return infos, errorList.Err() } // metaPageInfo returns page metadata for the meta page. @@ -1984,10 +2011,18 @@ func (tx *Tx) rootRecordPageInfo(pgno uint32) (*RootRecordPageInfo, error) { } func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error { - return tx.walkTree(root, 0, func(pgno, parent, typ uint32) error { + var errorList ErrorList + + if err := tx.walkTree(root, 0, func(pgno, parent, typ uint32, err error) error { + if err != nil { + errorList.Append(err) + return nil + } + buf, _, err := tx.readPage(pgno) if err != nil { - return err + errorList.Append(fmt.Errorf("cannot read page: pgno=%d parent=%d typ=%d err=%d", pgno, parent, typ, err)) + return nil } switch typ { @@ -2013,12 +2048,14 @@ func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error { Parent: parent, Tree: name, } - default: - vprint.PanicOn(fmt.Sprintf("unexpected page type %d for page %d", typ, pgno)) } return nil - }) + }); err != nil { + errorList.Append(err) + } + + return errorList.Err() } // PageData returns the raw page data for a single page. @@ -2042,6 +2079,20 @@ func (tx *Tx) GetSortedFieldViewList() (fvs []txkey.FieldView, _ error) { return } +func (tx *Tx) DebugInfo() *TxDebugInfo { + return &TxDebugInfo{ + Ptr: fmt.Sprintf("%p", tx), + Writable: tx.writable, + Stack: string(tx.stack), + } +} + +type TxDebugInfo struct { + Ptr string `json:"ptr"` + Writable bool `json:"writable"` + Stack string `json:"stack,omitempty"` +} + // SnapshotReader returns a reader that provides a snapshot for the current database state. func (tx *Tx) SnapshotReader() (io.Reader, error) { if tx.db == nil { diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 5be3a50ae..7a726c2ed 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "os" + "path/filepath" "strings" "sync" "testing" @@ -870,6 +871,36 @@ func TestTx_Check(t *testing.T) { t.Fatalf("unexpected error: %#v", err) } }) + + t.Run("ErrBadFreelist", func(t *testing.T) { + t.Parallel() + + db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-freelist")) + defer db.Close() + tx := MustBegin(t, db, false) + defer tx.Rollback() + + if err, ok := tx.Check().(rbf.ErrorList); !ok { + t.Fatal("expected error list") + } else if s := err.FullError(); !strings.Contains(s, `branch cell index out of range: pgno=2 i=0 n=0`) { + t.Fatalf("unexpected error:\n%s", s) + } + }) + + t.Run("ErrBadBitmap", func(t *testing.T) { + t.Parallel() + + db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-bitmap")) + defer db.Close() + tx := MustBegin(t, db, false) + defer tx.Rollback() + + if err, ok := tx.Check().(rbf.ErrorList); !ok { + t.Fatal("expected error list") + } else if s := err.FullError(); !strings.Contains(s, `cannot read page: pgno=65537 parent=3 err=rbf: page read out of bounds: pgno=65537 max=3`) { + t.Fatalf("unexpected error:\n%s", s) + } + }) } func mustReadPage(tb testing.TB, path string, pgno uint32) []byte { diff --git a/server.go b/server.go index 08b3b0fa5..36777bac8 100644 --- a/server.go +++ b/server.go @@ -514,6 +514,10 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.holder.schemator = s.schemator s.holder.sharder = s.sharder s.holder.serializer = s.serializer + + // Initial stats must be invoked after the executor obtains reference to the holder. + s.executor.InitStats() + return s, nil } diff --git a/server/handler_test.go b/server/handler_test.go index 283f102be..1e1105ff4 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -13,7 +13,6 @@ import ( gohttp "net/http" "net/http/httptest" "reflect" - "sort" "strings" "sync" "testing" @@ -24,10 +23,8 @@ import ( "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/pql" - pb "github.com/molecula/featurebase/v2/proto" "github.com/molecula/featurebase/v2/server" "github.com/molecula/featurebase/v2/test" - "google.golang.org/grpc" ) func TestHandler_PostSchemaCluster(t *testing.T) { @@ -1472,93 +1469,93 @@ func TestClusterTranslator(t *testing.T) { } } -func TestQueryHistory(t *testing.T) { - cluster := test.MustRunCluster(t, 3, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID("1"), - )}, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID("0"), - )}, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID("2"), - )}, - ) - defer cluster.Close() +// func TestQueryHistory(t *testing.T) { +// cluster := test.MustRunCluster(t, 3, +// []server.CommandOption{ +// server.OptCommandServerOptions( +// pilosa.OptServerNodeID("1"), +// )}, +// []server.CommandOption{ +// server.OptCommandServerOptions( +// pilosa.OptServerNodeID("0"), +// )}, +// []server.CommandOption{ +// server.OptCommandServerOptions( +// pilosa.OptServerNodeID("2"), +// )}, +// ) +// defer cluster.Close() - cmd := cluster.GetNode(0) - h := cmd.Handler.(*http.Handler).Handler +// cmd := cluster.GetNode(0) +// h := cmd.Handler.(*http.Handler).Handler - w := httptest.NewRecorder() +// w := httptest.NewRecorder() - test.Do(t, "POST", cmd.URL()+"/index/i0", "") - test.Do(t, "POST", cmd.URL()+"/index/i0/field/f0", "") +// test.Do(t, "POST", cmd.URL()+"/index/i0", "") +// test.Do(t, "POST", cmd.URL()+"/index/i0/field/f0", "") - gh := server.NewGRPCHandler(cmd.API) - stream := &MockServerTransportStream{} - ctx := grpc.NewContextWithServerTransportStream(context.Background(), stream) - _, err := gh.QuerySQLUnary(ctx, &pb.QuerySQLRequest{ - Sql: `select * from i0`, - }) +// gh := server.NewGRPCHandler(cmd.API) +// stream := &MockServerTransportStream{} +// ctx := grpc.NewContextWithServerTransportStream(context.Background(), stream) +// _, err := gh.QuerySQLUnary(ctx, &pb.QuerySQLRequest{ +// Sql: `select * from i0`, +// }) - if err != nil { - t.Fatalf("QuerySQLUnary failed: %v", err) - } +// if err != nil { +// t.Fatalf("QuerySQLUnary failed: %v", err) +// } - test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(0, f0=0)") - test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(3000000, f0=0)") - test.Do(t, "POST", cmd.URL()+"/index/i0/query", "TopN(f0)") +// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(0, f0=0)") +// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(3000000, f0=0)") +// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "TopN(f0)") - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/query-history", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) - } +// h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/query-history", nil)) +// if w.Code != gohttp.StatusOK { +// t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String()) +// } - ret := make([]pilosa.PastQueryStatus, 4) - b, err := ioutil.ReadAll(w.Body) - if err != nil { - t.Fatalf("reading: %v", err) - } - err = json.Unmarshal(b, &ret) - if err != nil { - t.Fatalf("unmarshalling: %v", err) - } +// ret := make([]pilosa.PastQueryStatus, 4) +// b, err := ioutil.ReadAll(w.Body) +// if err != nil { +// t.Fatalf("reading: %v", err) +// } +// err = json.Unmarshal(b, &ret) +// if err != nil { +// t.Fatalf("unmarshalling: %v", err) +// } - // verify result length - if len(ret) != 4 { - // each set query executes on both nodes once - // topn query gets added to history on node0 once, node1 twice - t.Fatalf("expected list of length 4, got %d\n%+v", len(ret), ret) - } +// // verify result length +// if len(ret) != 4 { +// // each set query executes on both nodes once +// // topn query gets added to history on node0 once, node1 twice +// t.Fatalf("expected list of length 4, got %d\n%+v", len(ret), ret) +// } - // verify sort order - if !sort.SliceIsSorted(ret, func(i, j int) bool { - // must match the sort in api.PastQueries - return ret[i].Start.After(ret[j].Start) - }) { - t.Fatalf("response list not sorted correctly") - } +// // verify sort order +// if !sort.SliceIsSorted(ret, func(i, j int) bool { +// // must match the sort in api.PastQueries +// return ret[i].Start.After(ret[j].Start) +// }) { +// t.Fatalf("response list not sorted correctly") +// } - // verify some response values - if ret[0].Index != "i0" { - t.Fatalf("response value for 'Index' was '%s', expected 'i0'", ret[0].Index) - } - if ret[0].Node != cluster.GetNode(0).Server.NodeID() { - t.Fatalf("response value for 'Node' was '%s', expected '%s'", ret[0].Node, cluster.GetNode(0).Server.NodeID()) - } - if ret[3].PQL != "Extract(All(),Rows(f0))" { - t.Fatalf("response value for 'PQL' was '%s', expected 'Extract(All(),Rows(f0))'", ret[0].PQL) - } - if ret[3].SQL != "select * from i0" { - t.Fatalf("response value for 'SQL' was '%s', expected 'select * from i0'", ret[0].SQL) - } - if ret[0].PQL != "TopN(f0)" { - t.Fatalf("response value for 'PQL' was '%s', expected 'TopN(f0)'", ret[0].PQL) - } -} +// // verify some response values +// if ret[0].Index != "i0" { +// t.Fatalf("response value for 'Index' was '%s', expected 'i0'", ret[0].Index) +// } +// if ret[0].Node != cluster.GetNode(0).Server.NodeID() { +// t.Fatalf("response value for 'Node' was '%s', expected '%s'", ret[0].Node, cluster.GetNode(0).Server.NodeID()) +// } +// if ret[3].PQL != "Extract(All(),Rows(f0))" { +// t.Fatalf("response value for 'PQL' was '%s', expected 'Extract(All(),Rows(f0))'", ret[0].PQL) +// } +// if ret[3].SQL != "select * from i0" { +// t.Fatalf("response value for 'SQL' was '%s', expected 'select * from i0'", ret[0].SQL) +// } +// if ret[0].PQL != "TopN(f0)" { +// t.Fatalf("response value for 'PQL' was '%s', expected 'TopN(f0)'", ret[0].PQL) +// } +// } func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) { dec := json.NewDecoder(r) diff --git a/server/server.go b/server/server.go index fd7475237..43b60f8f9 100644 --- a/server/server.go +++ b/server/server.go @@ -552,6 +552,15 @@ func (m *Command) SetupServer() error { m.querylogger.Infof("Group with admin level access: %v", p.Admin) m.querylogger.Infof("Permissions: %+v", p.Permissions) + + // disable postgres binding if auth is enabled + m.Config.Postgres.Bind = "" + + // TLS must be enabled if auth is + if m.Config.TLS.CertificatePath == "" || m.Config.TLS.CertificateKeyPath == "" || m.Config.TLS.CACertPath == "" { + return fmt.Errorf("transport layer security (TLS) is not configured properly. TLS is required when AuthN/Z is enabled, current configuration: %v", m.Config.TLS) + } + } m.Handler, err = http.NewHandler( diff --git a/server/server_test.go b/server/server_test.go index 014e2035e..551781ffb 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -17,7 +17,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/pql" @@ -26,6 +26,7 @@ import ( "github.com/molecula/featurebase/v2/test" "github.com/molecula/featurebase/v2/testhook" "github.com/pkg/errors" + "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ) @@ -504,6 +505,30 @@ func TestClusteringNodesReplica1(t *testing.T) { t.Fatalf("starting cluster: %v", err) } + indexName := "idx" + fieldName := "fld" + + // Create the schema. + if _, err := cluster.GetPrimary().API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil { + t.Fatalf("creating index: %v", err) + } + if _, err := cluster.GetPrimary().API.CreateField(context.Background(), indexName, fieldName); err != nil { + t.Fatalf("creating field: %v", err) + } + + // Set some columns across shards to ensure that the Row query will require + // data from all nodes. + data := []string{} + for rowID := 1; rowID < 2; rowID++ { + for columnID := 1; columnID < 10; columnID++ { + data = append(data, fmt.Sprintf(`Set(%d, %s=%d)`, columnID*pilosa.ShardWidth, fieldName, rowID)) + } + } + if _, err := cluster.GetPrimary().Query(t, indexName, "", strings.Join(data, "")); err != nil { + t.Fatalf("setting columns: %v", err) + } + + // Shut down a node. if err := cluster.GetNonPrimary().Command.Close(); err != nil { t.Fatalf("closing third node: %v", err) } @@ -513,7 +538,12 @@ func TestClusteringNodesReplica1(t *testing.T) { } // confirm that cluster stops accepting queries after one node closes - if _, err := cluster.GetPrimary().API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state DOWN") { + qry := &pilosa.QueryRequest{ + Index: "idx", + Query: fmt.Sprintf("Row(%s=1)", fieldName), + } + + if _, err := cluster.GetPrimary().API.Query(context.Background(), qry); !strings.Contains(err.Error(), "shard unavailable") { t.Fatalf("got unexpected error querying an incomplete cluster: %v", err) } } @@ -540,8 +570,34 @@ func TestClusteringNodesReplica2(t *testing.T) { } defer cluster.Close() + indexName := "idx" + fieldName := "fld" + coord, others := cluster.GetPrimary(), cluster.GetNonPrimaries() + // Create the schema. + if _, err := coord.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil { + t.Fatalf("creating index: %v", err) + } + if _, err := coord.API.CreateField(context.Background(), indexName, fieldName); err != nil { + t.Fatalf("creating field: %v", err) + } + + // Set some columns across shards to ensure that the Row query will require + // data from all nodes. + data := []string{} + cols := []uint64{} + for rowID := 1; rowID < 2; rowID++ { + for columnID := 1; columnID < 30; columnID++ { + col := uint64(columnID * pilosa.ShardWidth) + cols = append(cols, col) + data = append(data, fmt.Sprintf(`Set(%d, %s=%d)`, col, fieldName, rowID)) + } + } + if _, err := coord.Query(t, indexName, "", strings.Join(data, "")); err != nil { + t.Fatalf("setting columns: %v", err) + } + if err := others[0].Close(); err != nil { t.Fatalf("closing third node: %v", err) } @@ -569,8 +625,30 @@ func TestClusteringNodesReplica2(t *testing.T) { t.Fatalf("after closing second server: %v", err) } - if _, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state DOWN") { - t.Fatalf("got unexpected error querying an incomplete cluster: %v", err) + qry := &pilosa.QueryRequest{ + Index: "idx", + Query: fmt.Sprintf("Row(%s=1)", fieldName), + } + + // Because we no longer block queries when the cluster is in state DOWN, + // there are cases where a DOWN cluster can still respond to a query. In + // that case, we want the test to pass. But if the unavailable node(s) cause + // the query to result in an error, we check that it's the error we expect. + resp, err := coord.API.Query(context.Background(), qry) + if err != nil { + if !strings.Contains(err.Error(), "shard unavailable") { + t.Fatalf("got unexpected error querying an incomplete cluster: %v", err) + } + } else { + if len(resp.Results) == 0 { + t.Fatal("got no results") + } + + row, ok := resp.Results[0].(*pilosa.Row) + if !ok { + t.Fatalf("expected a *pilosa.Row, but got %T", resp.Results[0]) + } + require.Equal(t, row.Columns(), cols) } }