From 39f85fe2a4fd2439040f6ac7af07642d1daf966f Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 12:25:01 -0500 Subject: [PATCH 001/222] added terraform config for aws single node deployment --- .gitlab/.gitlab-ci.yml | 34 ++++++++++++++++++++++++ .terraform/main.tf | 58 +++++++++++++++++++++++++++++++++++++++++ .terraform/server.yml | 5 ++++ .terraform/variables.tf | 10 +++++++ 4 files changed, 107 insertions(+) create mode 100644 .terraform/main.tf create mode 100644 .terraform/server.yml create mode 100644 .terraform/variables.tf diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bb734ca27..599a8d65a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -2,10 +2,12 @@ include: - template: Security/SAST.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml + stages: - test - build + - deploy install lattice: stage: test @@ -137,3 +139,35 @@ build for darwin arm64: artifacts: paths: - featurebase_darwin_arm64 + +init terraform: + stage: deploy + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + before_script: + - cd .terraform + script: + - gitlab-terraform init + +validate terraform: + stage: deploy + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + script: + - gitlab-terraform validate + needs: + - job: init terraform + +plan terraform: + stage: deploy + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + script: + - gitlab-terraform plan + needs: + - job: validate terraform + +apply terraform: + stage: deploy + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + script: + - gitlab-terraform apply + needs: + - job: plan terraform \ No newline at end of file diff --git a/.terraform/main.tf b/.terraform/main.tf new file mode 100644 index 000000000..377853714 --- /dev/null +++ b/.terraform/main.tf @@ -0,0 +1,58 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.27" + } + } + + required_version = ">= 1.0.0" +} + +provider "aws" { + region = "us-east-2" +} + +resource "aws_security_group" "server_sg" { + name = "Load Balancer Security Group" + + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = -1 + cidr_blocks = ["0.0.0.0/0"] + } +} + +data "cloudinit_config" "server_config" { + gzip = true + base64_encode = true + part { + content_type = "text/cloud-config" + content = templatefile("${path.module}/server.yml", { + header : aws_security_group.server_sg.id + }) + } +} + +resource "aws_instance" "server_instance1" { + ami = var.ami_linux_amd + instance_type = "t2.micro" + + key_name = "souhaila-rsa" + + vpc_security_group_ids = [aws_security_group.server_sg.id] + user_data = data.cloudinit_config.server_config.rendered + associate_public_ip_address = true + + tags = { + Name = var.instance_name + } +} \ No newline at end of file diff --git a/.terraform/server.yml b/.terraform/server.yml new file mode 100644 index 000000000..5c7571b83 --- /dev/null +++ b/.terraform/server.yml @@ -0,0 +1,5 @@ +#cloud-config +runcmd: + - sudo yum update + - sudo yum install postgresql14 + - sudo yum install golang1.16 \ No newline at end of file diff --git a/.terraform/variables.tf b/.terraform/variables.tf new file mode 100644 index 000000000..2d1859c98 --- /dev/null +++ b/.terraform/variables.tf @@ -0,0 +1,10 @@ +variable "instance_name" { + description = "Value of the Name tag for the EC2 instance" + type = string + default = "single-node-deployment-1" +} + +variable "ami_linux_amd" { + type = string + default = "ami-0f19d220602031aed" +} From d565303160f28483d4b53f000517ace6a7c3865c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 13:28:33 -0500 Subject: [PATCH 002/222] fix script not running in correct directory --- .gitlab/.gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 599a8d65a..67f2250a8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,6 +151,8 @@ init terraform: validate terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + before_script: + - cd .terraform script: - gitlab-terraform validate needs: @@ -159,6 +161,8 @@ validate terraform: plan terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + before_script: + - cd .terraform script: - gitlab-terraform plan needs: @@ -167,6 +171,8 @@ plan terraform: apply terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + before_script: + - cd .terraform script: - gitlab-terraform apply needs: From 01e4ae40700a3f583c41e7d3d8f422b6aa033b0f Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 14:13:53 -0500 Subject: [PATCH 003/222] fix missing plan file for terraform apply --- .gitlab/.gitlab-ci.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 67f2250a8..fc415a28b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -3,7 +3,6 @@ include: - template: Security/License-Scanning.gitlab-ci.yml - template: Security/Dependency-Scanning.gitlab-ci.yml - stages: - test - build @@ -147,6 +146,9 @@ init terraform: - cd .terraform script: - gitlab-terraform init + cache: + paths: + - .terraform validate terraform: stage: deploy @@ -157,6 +159,9 @@ validate terraform: - gitlab-terraform validate needs: - job: init terraform + cache: + paths: + - .terraform plan terraform: stage: deploy @@ -164,9 +169,15 @@ plan terraform: before_script: - cd .terraform script: - - gitlab-terraform plan + - gitlab-terraform plan -out=plan.tfplan needs: - job: validate terraform + cache: + paths: + - .terraform + artifacts: + paths: + - plan.tfplan apply terraform: stage: deploy @@ -174,6 +185,9 @@ apply terraform: before_script: - cd .terraform script: - - gitlab-terraform apply + - gitlab-terraform apply -input=false plan.tfplan needs: - - job: plan terraform \ No newline at end of file + - job: plan terraform + cache: + paths: + - .terraform \ No newline at end of file From f3884e21bafdd094daa81572d3bdb78f79d65e1b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 14:40:06 -0500 Subject: [PATCH 004/222] changed directory name for terraform files --- .gitlab/.gitlab-ci.yml | 8 ++++---- {.terraform => molecular-terraform}/main.tf | 0 {.terraform => molecular-terraform}/server.yml | 0 {.terraform => molecular-terraform}/variables.tf | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename {.terraform => molecular-terraform}/main.tf (100%) rename {.terraform => molecular-terraform}/server.yml (100%) rename {.terraform => molecular-terraform}/variables.tf (100%) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index fc415a28b..d109853f0 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -143,7 +143,7 @@ init terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: - - cd .terraform + - cd molecular-terraform script: - gitlab-terraform init cache: @@ -154,7 +154,7 @@ validate terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: - - cd .terraform + - cd molecular-terraform script: - gitlab-terraform validate needs: @@ -167,7 +167,7 @@ plan terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: - - cd .terraform + - cd molecular-terraform script: - gitlab-terraform plan -out=plan.tfplan needs: @@ -183,7 +183,7 @@ apply terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: - - cd .terraform + - cd molecular-terraform script: - gitlab-terraform apply -input=false plan.tfplan needs: diff --git a/.terraform/main.tf b/molecular-terraform/main.tf similarity index 100% rename from .terraform/main.tf rename to molecular-terraform/main.tf diff --git a/.terraform/server.yml b/molecular-terraform/server.yml similarity index 100% rename from .terraform/server.yml rename to molecular-terraform/server.yml diff --git a/.terraform/variables.tf b/molecular-terraform/variables.tf similarity index 100% rename from .terraform/variables.tf rename to molecular-terraform/variables.tf From a8d3d1c4d4a0790bf7471ae7b6f235de53180e95 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 2 Nov 2021 15:00:58 -0500 Subject: [PATCH 005/222] fix terraform apply inputs --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d109853f0..68be7fb55 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -143,6 +143,7 @@ init terraform: stage: deploy image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest before_script: + - rm -rf .terraform - cd molecular-terraform script: - gitlab-terraform init @@ -185,7 +186,7 @@ apply terraform: before_script: - cd molecular-terraform script: - - gitlab-terraform apply -input=false plan.tfplan + - gitlab-terraform apply needs: - job: plan terraform cache: From c29dcddca6846e205b15d59c78afe8bdf5f7c0f5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 08:57:10 -0500 Subject: [PATCH 006/222] replace terraform with awscli for integration test --- .gitlab/.gitlab-ci.yml | 70 +++++++-------------- {molecular-terraform => .gitlab}/server.yml | 0 molecular-terraform/main.tf | 58 ----------------- molecular-terraform/variables.tf | 10 --- 4 files changed, 21 insertions(+), 117 deletions(-) rename {molecular-terraform => .gitlab}/server.yml (100%) mode change 100644 => 100755 delete mode 100644 molecular-terraform/main.tf delete mode 100644 molecular-terraform/variables.tf diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 68be7fb55..f12c60249 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -6,7 +6,7 @@ include: stages: - test - build - - deploy + - integration install lattice: stage: test @@ -139,56 +139,28 @@ build for darwin arm64: paths: - featurebase_darwin_arm64 -init terraform: - stage: deploy - image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest +deploy ec2 instance: + stage: integration + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + AMI: "" + INSTANCE: "t2_micro" + SECURITY_GROUP: "" + SUBNET_ID: "" + SSH_KEY: "" + PUBLIC_IP: "" before_script: - - rm -rf .terraform - - cd molecular-terraform + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - export AWS_PROFILE=gitlab-cicd script: - - gitlab-terraform init - cache: - paths: - - .terraform + - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile gitlab-cicd) -validate terraform: - stage: deploy - image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest - before_script: - - cd molecular-terraform - script: - - gitlab-terraform validate - needs: - - job: init terraform - cache: - paths: - - .terraform + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile gitlab-cicd) -plan terraform: - stage: deploy - image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest - before_script: - - cd molecular-terraform - script: - - gitlab-terraform plan -out=plan.tfplan - needs: - - job: validate terraform - cache: - paths: - - .terraform - artifacts: - paths: - - plan.tfplan + - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile gitlab-cicd) -apply terraform: - stage: deploy - image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest - before_script: - - cd molecular-terraform - script: - - gitlab-terraform apply - needs: - - job: plan terraform - cache: - paths: - - .terraform \ No newline at end of file + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID —user-data file://.gitlab/server.yml + + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile gitlab-cicd) \ No newline at end of file diff --git a/molecular-terraform/server.yml b/.gitlab/server.yml old mode 100644 new mode 100755 similarity index 100% rename from molecular-terraform/server.yml rename to .gitlab/server.yml diff --git a/molecular-terraform/main.tf b/molecular-terraform/main.tf deleted file mode 100644 index 377853714..000000000 --- a/molecular-terraform/main.tf +++ /dev/null @@ -1,58 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 3.27" - } - } - - required_version = ">= 1.0.0" -} - -provider "aws" { - region = "us-east-2" -} - -resource "aws_security_group" "server_sg" { - name = "Load Balancer Security Group" - - ingress { - from_port = 80 - to_port = 80 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = -1 - cidr_blocks = ["0.0.0.0/0"] - } -} - -data "cloudinit_config" "server_config" { - gzip = true - base64_encode = true - part { - content_type = "text/cloud-config" - content = templatefile("${path.module}/server.yml", { - header : aws_security_group.server_sg.id - }) - } -} - -resource "aws_instance" "server_instance1" { - ami = var.ami_linux_amd - instance_type = "t2.micro" - - key_name = "souhaila-rsa" - - vpc_security_group_ids = [aws_security_group.server_sg.id] - user_data = data.cloudinit_config.server_config.rendered - associate_public_ip_address = true - - tags = { - Name = var.instance_name - } -} \ No newline at end of file diff --git a/molecular-terraform/variables.tf b/molecular-terraform/variables.tf deleted file mode 100644 index 2d1859c98..000000000 --- a/molecular-terraform/variables.tf +++ /dev/null @@ -1,10 +0,0 @@ -variable "instance_name" { - description = "Value of the Name tag for the EC2 instance" - type = string - default = "single-node-deployment-1" -} - -variable "ami_linux_amd" { - type = string - default = "ami-0f19d220602031aed" -} From e1ca09a7b5dd995910e59d09d4c16e07dab2a991 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 09:02:17 -0500 Subject: [PATCH 007/222] comment other parts of script to speed up testing --- .gitlab/.gitlab-ci.yml | 244 ++++++++++++++++++++--------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f12c60249..39f12afcb 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -4,140 +4,140 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml stages: - - test - - build + # - test + # - build - integration -install lattice: - stage: test - image: node:14 - script: - - echo "Installing lattice dependencies..." - - cd lattice - - yarn install +# install lattice: +# stage: test +# image: node:14 +# script: +# - echo "Installing lattice dependencies..." +# - cd lattice +# - yarn install -build lattice: - stage: test - image: node:14 - variables: - CI: "true" - script: - - echo "Building lattice..." - - cd lattice - - yarn build - - tar -czvf lattice.tar.gz lattice/build - artifacts: - paths: - - lattice/lattice.tar.gz - needs: - - job: install lattice - allow_failure: true +# build lattice: +# stage: test +# image: node:14 +# variables: +# CI: "true" +# script: +# - echo "Building lattice..." +# - cd lattice +# - yarn build +# - tar -czvf lattice.tar.gz lattice/build +# artifacts: +# paths: +# - lattice/lattice.tar.gz +# needs: +# - job: install lattice +# allow_failure: true -run jest tests: - stage: test - image: node:14 - variables: - CI: "true" - script: - - echo "Testing lattice..." - - cd lattice - - npm install --force - - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter - artifacts: - paths: - - lattice/coverage/lcov.info +# run jest tests: +# stage: test +# image: node:14 +# variables: +# CI: "true" +# script: +# - echo "Testing lattice..." +# - cd lattice +# - npm install --force +# - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter +# artifacts: +# paths: +# - lattice/coverage/lcov.info -run go tests: - stage: test - image: golang:1.16.9 - script: - - echo "Running featurebase unit tests..." - - go test -covermode=atomic -coverprofile=coverage.out - artifacts: - paths: - - coverage.out +# run go tests: +# stage: test +# image: golang:1.16.9 +# script: +# - echo "Running featurebase unit tests..." +# - go test -covermode=atomic -coverprofile=coverage.out +# artifacts: +# paths: +# - coverage.out -run go tests with output: - stage: test - image: golang:1.16.9 - script: - - echo "Running featurebase unit tests to capture JSON output..." - - go test -json > test-report.out - artifacts: - paths: - - test-report.out +# run go tests with output: +# stage: test +# image: golang:1.16.9 +# script: +# - echo "Running featurebase unit tests to capture JSON output..." +# - go test -json > test-report.out +# artifacts: +# paths: +# - test-report.out -generate coverage html: - stage: test - image: golang:1.16.9 - script: - - echo "Generating coverage HTML.." - - go tool cover -html coverage.out -o coverage.html - artifacts: - paths: - - coverage.html - needs: - - job: run go tests +# generate coverage html: +# stage: test +# image: golang:1.16.9 +# script: +# - echo "Generating coverage HTML.." +# - go tool cover -html coverage.out -o coverage.html +# artifacts: +# paths: +# - coverage.html +# needs: +# - job: run go tests -upload to sonarcloud: - stage: test - image: sonarsource/sonar-scanner-cli:4.6 - variables: - SONAR_TOKEN: $SONAR_TOKEN - script: - - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info - needs: - - job: run go tests - - job: run go tests with output - - job: run jest tests +# upload to sonarcloud: +# stage: test +# image: sonarsource/sonar-scanner-cli:4.6 +# variables: +# SONAR_TOKEN: $SONAR_TOKEN +# script: +# - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info +# needs: +# - job: run go tests +# - job: run go tests with output +# - job: run jest tests -build for linux amd64: - stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - variables: - GOOS: "linux" - GOARCH: "amd64" - script: - - go build -o featurebase_linux_amd64 ./cmd/featurebase - artifacts: - paths: - - featurebase_linux_amd64 +# build for linux amd64: +# stage: build +# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 +# variables: +# GOOS: "linux" +# GOARCH: "amd64" +# script: +# - go build -o featurebase_linux_amd64 ./cmd/featurebase +# artifacts: +# paths: +# - featurebase_linux_amd64 -build for linux arm64: - stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - variables: - GOOS: "linux" - GOARCH: "arm64" - script: - - go build -o featurebase_linux_arm64 ./cmd/featurebase - artifacts: - paths: - - featurebase_linux_arm64 +# build for linux arm64: +# stage: build +# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 +# variables: +# GOOS: "linux" +# GOARCH: "arm64" +# script: +# - go build -o featurebase_linux_arm64 ./cmd/featurebase +# artifacts: +# paths: +# - featurebase_linux_arm64 -build for darwin amd64: - stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - variables: - GOOS: "darwin" - GOARCH: "amd64" - script: - - go build -o featurebase_darwin_amd64 ./cmd/featurebase - artifacts: - paths: - - featurebase_darwin_amd64 +# build for darwin amd64: +# stage: build +# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 +# variables: +# GOOS: "darwin" +# GOARCH: "amd64" +# script: +# - go build -o featurebase_darwin_amd64 ./cmd/featurebase +# artifacts: +# paths: +# - featurebase_darwin_amd64 -build for darwin arm64: - stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - variables: - GOOS: "darwin" - GOARCH: "arm64" - script: - - go build -o featurebase_darwin_arm64 ./cmd/featurebase - artifacts: - paths: - - featurebase_darwin_arm64 +# build for darwin arm64: +# stage: build +# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 +# variables: +# GOOS: "darwin" +# GOARCH: "arm64" +# script: +# - go build -o featurebase_darwin_arm64 ./cmd/featurebase +# artifacts: +# paths: +# - featurebase_darwin_arm64 deploy ec2 instance: stage: integration From 6ee2341c8810f15b17339e56edb39cd1907d31c6 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 09:04:43 -0500 Subject: [PATCH 008/222] undo last commit changes --- .gitlab/.gitlab-ci.yml | 244 ++++++++++++++++++++--------------------- 1 file changed, 122 insertions(+), 122 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 39f12afcb..f12c60249 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -4,140 +4,140 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml stages: - # - test - # - build + - test + - build - integration -# install lattice: -# stage: test -# image: node:14 -# script: -# - echo "Installing lattice dependencies..." -# - cd lattice -# - yarn install +install lattice: + stage: test + image: node:14 + script: + - echo "Installing lattice dependencies..." + - cd lattice + - yarn install -# build lattice: -# stage: test -# image: node:14 -# variables: -# CI: "true" -# script: -# - echo "Building lattice..." -# - cd lattice -# - yarn build -# - tar -czvf lattice.tar.gz lattice/build -# artifacts: -# paths: -# - lattice/lattice.tar.gz -# needs: -# - job: install lattice -# allow_failure: true +build lattice: + stage: test + image: node:14 + variables: + CI: "true" + script: + - echo "Building lattice..." + - cd lattice + - yarn build + - tar -czvf lattice.tar.gz lattice/build + artifacts: + paths: + - lattice/lattice.tar.gz + needs: + - job: install lattice + allow_failure: true -# run jest tests: -# stage: test -# image: node:14 -# variables: -# CI: "true" -# script: -# - echo "Testing lattice..." -# - cd lattice -# - npm install --force -# - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter -# artifacts: -# paths: -# - lattice/coverage/lcov.info +run jest tests: + stage: test + image: node:14 + variables: + CI: "true" + script: + - echo "Testing lattice..." + - cd lattice + - npm install --force + - npm test -- --coverage --testResultsProcessor=jest-sonar-reporter + artifacts: + paths: + - lattice/coverage/lcov.info -# run go tests: -# stage: test -# image: golang:1.16.9 -# script: -# - echo "Running featurebase unit tests..." -# - go test -covermode=atomic -coverprofile=coverage.out -# artifacts: -# paths: -# - coverage.out +run go tests: + stage: test + image: golang:1.16.9 + script: + - echo "Running featurebase unit tests..." + - go test -covermode=atomic -coverprofile=coverage.out + artifacts: + paths: + - coverage.out -# run go tests with output: -# stage: test -# image: golang:1.16.9 -# script: -# - echo "Running featurebase unit tests to capture JSON output..." -# - go test -json > test-report.out -# artifacts: -# paths: -# - test-report.out +run go tests with output: + stage: test + image: golang:1.16.9 + script: + - echo "Running featurebase unit tests to capture JSON output..." + - go test -json > test-report.out + artifacts: + paths: + - test-report.out -# generate coverage html: -# stage: test -# image: golang:1.16.9 -# script: -# - echo "Generating coverage HTML.." -# - go tool cover -html coverage.out -o coverage.html -# artifacts: -# paths: -# - coverage.html -# needs: -# - job: run go tests +generate coverage html: + stage: test + image: golang:1.16.9 + script: + - echo "Generating coverage HTML.." + - go tool cover -html coverage.out -o coverage.html + artifacts: + paths: + - coverage.html + needs: + - job: run go tests -# upload to sonarcloud: -# stage: test -# image: sonarsource/sonar-scanner-cli:4.6 -# variables: -# SONAR_TOKEN: $SONAR_TOKEN -# script: -# - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info -# needs: -# - job: run go tests -# - job: run go tests with output -# - job: run jest tests +upload to sonarcloud: + stage: test + image: sonarsource/sonar-scanner-cli:4.6 + variables: + SONAR_TOKEN: $SONAR_TOKEN + script: + - sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info + needs: + - job: run go tests + - job: run go tests with output + - job: run jest tests -# build for linux amd64: -# stage: build -# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 -# variables: -# GOOS: "linux" -# GOARCH: "amd64" -# script: -# - go build -o featurebase_linux_amd64 ./cmd/featurebase -# artifacts: -# paths: -# - featurebase_linux_amd64 +build for linux amd64: + stage: build + image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + variables: + GOOS: "linux" + GOARCH: "amd64" + script: + - go build -o featurebase_linux_amd64 ./cmd/featurebase + artifacts: + paths: + - featurebase_linux_amd64 -# build for linux arm64: -# stage: build -# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 -# variables: -# GOOS: "linux" -# GOARCH: "arm64" -# script: -# - go build -o featurebase_linux_arm64 ./cmd/featurebase -# artifacts: -# paths: -# - featurebase_linux_arm64 +build for linux arm64: + stage: build + image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + variables: + GOOS: "linux" + GOARCH: "arm64" + script: + - go build -o featurebase_linux_arm64 ./cmd/featurebase + artifacts: + paths: + - featurebase_linux_arm64 -# build for darwin amd64: -# stage: build -# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 -# variables: -# GOOS: "darwin" -# GOARCH: "amd64" -# script: -# - go build -o featurebase_darwin_amd64 ./cmd/featurebase -# artifacts: -# paths: -# - featurebase_darwin_amd64 +build for darwin amd64: + stage: build + image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + variables: + GOOS: "darwin" + GOARCH: "amd64" + script: + - go build -o featurebase_darwin_amd64 ./cmd/featurebase + artifacts: + paths: + - featurebase_darwin_amd64 -# build for darwin arm64: -# stage: build -# image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 -# variables: -# GOOS: "darwin" -# GOARCH: "arm64" -# script: -# - go build -o featurebase_darwin_arm64 ./cmd/featurebase -# artifacts: -# paths: -# - featurebase_darwin_arm64 +build for darwin arm64: + stage: build + image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + variables: + GOOS: "darwin" + GOARCH: "arm64" + script: + - go build -o featurebase_darwin_arm64 ./cmd/featurebase + artifacts: + paths: + - featurebase_darwin_arm64 deploy ec2 instance: stage: integration From df2064a860f266781e35cad1f20f4306183bcc05 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 09:31:31 -0500 Subject: [PATCH 009/222] define aws credential variables --- .gitlab/.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f12c60249..fc97e4e78 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -149,6 +149,8 @@ deploy ec2 instance: SUBNET_ID: "" SSH_KEY: "" PUBLIC_IP: "" + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY From 50f409771fe7fbe7bf9b9a9eda0fb5beaa5c8502 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 09:59:11 -0500 Subject: [PATCH 010/222] updated aws profile --- .gitlab/.gitlab-ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index fc97e4e78..8091bffdc 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,18 +151,19 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_PROFILE: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - export AWS_PROFILE=gitlab-cicd + - AWS_PROFILE=$(aws configure list-profiles) script: - - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile gitlab-cicd) + - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $AWS_PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile gitlab-cicd) + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $AWS_PROFILE) - - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile gitlab-cicd) + - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $AWS_PROFILE) - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID —user-data file://.gitlab/server.yml - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile gitlab-cicd) \ No newline at end of file + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $AWS_PROFILE) \ No newline at end of file From b95d013e14bb74bd2b04f5aa3ee777b2d0ff1a80 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 10:12:26 -0500 Subject: [PATCH 011/222] configure aws profile --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8091bffdc..d7f5cf3c1 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,12 +151,12 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - AWS_PROFILE: "" + AWS_PROFILE: "AWS-PROFILE" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - AWS_PROFILE=$(aws configure list-profiles) + - aws configure --profile $AWS_PROFILE script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $AWS_PROFILE) From c19aa7d6b31f83d1215b39709aaef5385d2975c8 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 10:27:13 -0500 Subject: [PATCH 012/222] changed profile name --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d7f5cf3c1..aba3baf19 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,7 +151,7 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - AWS_PROFILE: "AWS-PROFILE" + AWS_PROFILE: "integration" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY From 6e673bafa37218879fc736f376ccae837a4035d1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 10:58:08 -0500 Subject: [PATCH 013/222] configure profile --- .gitlab/.gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index aba3baf19..8cf121587 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,19 +151,19 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - AWS_PROFILE: "integration" + PROFILE: "integration" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - aws configure --profile $AWS_PROFILE + - aws configure set aws_profile $PROFILE script: - - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $AWS_PROFILE) + - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $AWS_PROFILE) + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $AWS_PROFILE) + - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID —user-data file://.gitlab/server.yml - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $AWS_PROFILE) \ No newline at end of file + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) \ No newline at end of file From 67da6d9ffa817fb2132a62c00c1982c912a617fc Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 11:12:30 -0500 Subject: [PATCH 014/222] debug profile --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8cf121587..e2b33934c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -157,6 +157,7 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE + - aws configure list-profiles script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 502f40593e599fdbbfe121f32f93ce9eca913cee Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 11:26:05 -0500 Subject: [PATCH 015/222] updated profile --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e2b33934c..d653d362c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,13 +151,13 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - PROFILE: "integration" + PROFILE: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - aws configure list-profiles + - PROFILE=$(aws configure list-profiles) script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 01260cfcced82431a16502c07bc073e5476dc7af Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 11:38:58 -0500 Subject: [PATCH 016/222] update profile --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d653d362c..421a7b5ff 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -151,7 +151,7 @@ deploy ec2 instance: PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY - PROFILE: "" + PROFILE: "default" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY From 7a6e7c2fd5625fb0cf648a78f8b162b407677f68 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 11:53:43 -0500 Subject: [PATCH 017/222] set profile --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 421a7b5ff..f6a8ed451 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -158,6 +158,7 @@ deploy ec2 instance: - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - PROFILE=$(aws configure list-profiles) + - aws sts get-caller-identity --profile $PROFILE script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 8274a321277a99f2573042934c0b342e0e13657c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 12:09:20 -0500 Subject: [PATCH 018/222] debug profile --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f6a8ed451..2f3810c62 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -157,7 +157,8 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - PROFILE=$(aws configure list-profiles) + - PROFILE=$(aws configure list-profiles --output text) + - aws configure list-profiles --output text - aws sts get-caller-identity --profile $PROFILE script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 20224b1df5afac0683cd76baa787bd54180a1843 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 13:28:48 -0500 Subject: [PATCH 019/222] updated aws config --- .gitlab/.gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 2f3810c62..303d69af0 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -149,8 +149,8 @@ deploy ec2 instance: SUBNET_ID: "" SSH_KEY: "" PUBLIC_IP: "" - AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_ACCESS_KEY_ID: $AWS_CLOUD_DEV_KEY + AWS_SECRET_ACCESS_KEY: $AWS_CLOUD_DEV_SECRET PROFILE: "default" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID @@ -159,7 +159,6 @@ deploy ec2 instance: - aws configure set aws_profile $PROFILE - PROFILE=$(aws configure list-profiles --output text) - aws configure list-profiles --output text - - aws sts get-caller-identity --profile $PROFILE script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 08161a366af47079b3c6219263fd43bd1fe0305b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 13:42:03 -0500 Subject: [PATCH 020/222] undo last commit --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 303d69af0..8c6bf9c1c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -149,8 +149,8 @@ deploy ec2 instance: SUBNET_ID: "" SSH_KEY: "" PUBLIC_IP: "" - AWS_ACCESS_KEY_ID: $AWS_CLOUD_DEV_KEY - AWS_SECRET_ACCESS_KEY: $AWS_CLOUD_DEV_SECRET + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID From f80f2fc702b66f7af81b1a11a0cb0e55540893f6 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 14:34:19 -0500 Subject: [PATCH 021/222] added rsa key --- .gitlab/.gitlab-ci.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8c6bf9c1c..59bf3488e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -152,6 +152,7 @@ deploy ec2 instance: AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" + RSA_KEY: $AWS_RSA_Private_Key before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY @@ -161,11 +162,8 @@ deploy ec2 instance: - aws configure list-profiles --output text script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID —user-data file://.gitlab/server.yml - + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID -—user-data file://.gitlab/server.yml --key-name $RSA_KEY + - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) \ No newline at end of file From 82b97022ff9559322aea3222df02df29a238372f Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 4 Nov 2021 14:57:41 -0500 Subject: [PATCH 022/222] removed sleep time --- .gitlab/.gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a434f30b6..95a9c0af8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -202,5 +202,4 @@ deploy ec2 instance: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID -—user-data file://.gitlab/server.yml --key-name $RSA_KEY - - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) From 321d273dce080593b92fd8c0dd1eb723a7b8bab0 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:03:51 -0500 Subject: [PATCH 023/222] fixed typo in instance name --- .gitlab/.gitlab-ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 95a9c0af8..11182d7c8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -177,11 +177,11 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy ec2 instance: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" - INSTANCE: "t2_micro" + INSTANCE: "t2.micro" SECURITY_GROUP: "" SUBNET_ID: "" SSH_KEY: "" @@ -189,7 +189,6 @@ deploy ec2 instance: AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" - RSA_KEY: $AWS_RSA_Private_Key before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY @@ -201,5 +200,5 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID -—user-data file://.gitlab/server.yml --key-name $RSA_KEY - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) \ No newline at end of file From 383663673ba55de953fbe2c334266eddb4368c5d Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:11:27 -0500 Subject: [PATCH 024/222] configure ssh --- .gitlab/.gitlab-ci.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 11182d7c8..c80bef661 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -13,6 +13,15 @@ include: variables: GOVERSION: "1.16.9" +before_script: + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval `ssh-agent -s` + - mkdir -p /root/.ssh + - chmod 700 /root/.ssh + - cp $AWS_RSA_PRIVATE_KEY /root/.ssh/id_rsa + - chmod 600 /root/.ssh/id_rsa + - ssh-add /root/.ssh/id_rsa + stages: - lint - test @@ -200,5 +209,6 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) \ No newline at end of file + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-pair "AWS_RSA_PRIVATE_KEY" + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + - echo $PUBLIC_IP \ No newline at end of file From f9ff3973d0675ee439a29ab376aa14cb340bb1db Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:17:19 -0500 Subject: [PATCH 025/222] fixed argument name --- .gitlab/.gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c80bef661..fde2c6c56 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -198,6 +198,7 @@ deploy ec2 instance: AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" + INSTANCE_NAME: "Single_Node_Deployment" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY @@ -209,6 +210,7 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-pair "AWS_RSA_PRIVATE_KEY" + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name "AWS_RSA_PRIVATE_KEY" --tag-specifications \ + 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 14a9662494a868a7b779d2f7b9116ff581897e6c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:24:20 -0500 Subject: [PATCH 026/222] fixed yaml syntax error --- .gitlab/.gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index fde2c6c56..2b559da53 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -210,7 +210,6 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name "AWS_RSA_PRIVATE_KEY" --tag-specifications \ - 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name "AWS_RSA_PRIVATE_KEY" --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 7271821ebea0250d2eb437914b1b5c2fb924fc33 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:28:25 -0500 Subject: [PATCH 027/222] fixed path for ssh keys --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 2b559da53..2b4c8cc9d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -210,6 +210,6 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name "AWS_RSA_PRIVATE_KEY" --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name /root/.ssh/id_rsa --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From d147c03e227f10765a0a04079c5b681af5752f00 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:32:53 -0500 Subject: [PATCH 028/222] add ssh config in section with using ssh keys --- .gitlab/.gitlab-ci.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 2b4c8cc9d..ef3ac1d37 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -13,15 +13,6 @@ include: variables: GOVERSION: "1.16.9" -before_script: - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - - eval `ssh-agent -s` - - mkdir -p /root/.ssh - - chmod 700 /root/.ssh - - cp $AWS_RSA_PRIVATE_KEY /root/.ssh/id_rsa - - chmod 600 /root/.ssh/id_rsa - - ssh-add /root/.ssh/id_rsa - stages: - lint - test @@ -193,7 +184,6 @@ deploy ec2 instance: INSTANCE: "t2.micro" SECURITY_GROUP: "" SUBNET_ID: "" - SSH_KEY: "" PUBLIC_IP: "" AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY @@ -206,6 +196,13 @@ deploy ec2 instance: - aws configure set aws_profile $PROFILE - PROFILE=$(aws configure list-profiles --output text) - aws configure list-profiles --output text + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval `ssh-agent -s` + - mkdir -p /root/.ssh + - chmod 700 /root/.ssh + - cp $AWS_RSA_PRIVATE_KEY /root/.ssh/id_rsa + - chmod 600 /root/.ssh/id_rsa + - ssh-add /root/.ssh/id_rsa script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) From bd440f4c23baf61ebd77eb385bc8f02f086e4cca Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:43:19 -0500 Subject: [PATCH 029/222] updated permissions for ssh key --- .gitlab/.gitlab-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ef3ac1d37..634e07765 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,13 +200,11 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh - - cp $AWS_RSA_PRIVATE_KEY /root/.ssh/id_rsa - - chmod 600 /root/.ssh/id_rsa - - ssh-add /root/.ssh/id_rsa + - chmod 400 $AWS_SSH_PRIVATE_KEY script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name /root/.ssh/id_rsa --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name $AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 04b64c0f301c835cc254ea5f6e507df3e44e82d7 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 09:54:37 -0500 Subject: [PATCH 030/222] add path for ssh keys --- .gitlab/.gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 634e07765..66e04b2a9 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,11 +200,12 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh - - chmod 400 $AWS_SSH_PRIVATE_KEY + - cp $AWS_SSH_PRIVATE_KEY /builds/molecula/featurebase.tmp/ + - chmod 400 /builds/molecula/featurebase.tmp/$AWS_SSH_PRIVATE_KEY script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name $AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name /builds/molecula/featurebase.tmp/$AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 6643e3963e82516f5f8549e2dcfb5a9f28f2aed1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:28:36 -0500 Subject: [PATCH 031/222] reverted back to variable for ssh --- .gitlab/.gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 66e04b2a9..8591d9728 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -189,6 +189,7 @@ deploy ec2 instance: AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" INSTANCE_NAME: "Single_Node_Deployment" + AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY @@ -200,12 +201,10 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh - - cp $AWS_SSH_PRIVATE_KEY /builds/molecula/featurebase.tmp/ - - chmod 400 /builds/molecula/featurebase.tmp/$AWS_SSH_PRIVATE_KEY script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name /builds/molecula/featurebase.tmp/$AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name $AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From f7497bfe8ffba6ec1f8fed1b404c6ce8ba25fdef Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:35:15 -0500 Subject: [PATCH 032/222] pass ssh key to a file --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8591d9728..397e6ba0d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -201,10 +201,11 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh + - echo $AWS_SSH_PRIVATE_KEY > aws_ssh_key script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name $AWS_SSH_PRIVATE_KEY --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 550a610b5be50fba4de67cc4aef36b7e490fa1a9 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:46:54 -0500 Subject: [PATCH 033/222] sanity checks --- .gitlab/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 397e6ba0d..5e3343f4a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,8 +195,7 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - PROFILE=$(aws configure list-profiles --output text) - - aws configure list-profiles --output text + - echo $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh @@ -206,6 +205,7 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --count 1 --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - echo $AMI, $SECURITY_GROUP, $SUBNET_ID + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 0cfd5f116d171f029d90bf72dac6f3a471d4467b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:50:59 -0500 Subject: [PATCH 034/222] echo ssh file --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 5e3343f4a..065e20bbc 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,7 +195,6 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - echo $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh @@ -206,6 +205,7 @@ deploy ec2 instance: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - echo $AMI, $SECURITY_GROUP, $SUBNET_ID + - cat aws_ssh_key - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 021af2c5e5073b06b016e5e683ab1a1f56b2522c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 10:56:10 -0500 Subject: [PATCH 035/222] changed ssh key name to match aws --- .gitlab/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 065e20bbc..897119510 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -199,13 +199,13 @@ deploy ec2 instance: - eval `ssh-agent -s` - mkdir -p /root/.ssh - chmod 700 /root/.ssh - - echo $AWS_SSH_PRIVATE_KEY > aws_ssh_key + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - echo $AMI, $SECURITY_GROUP, $SUBNET_ID - - cat aws_ssh_key - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name aws_ssh_key --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - cat gitlab-featurebase-dev + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From a9ab597d28cba8e39e995580794416d99ac51ed5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 11:20:21 -0500 Subject: [PATCH 036/222] added profile to start-instance --- .gitlab/.gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 897119510..6f846c13a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -204,8 +204,6 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - echo $AMI, $SECURITY_GROUP, $SUBNET_ID - - cat gitlab-featurebase-dev - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' --profile $PROFILE - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP \ No newline at end of file From 6e6d7a77db68baffe0d52fbe8f0a7cf19c42ec82 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 11:26:09 -0500 Subject: [PATCH 037/222] debug echo statements --- .gitlab/.gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6f846c13a..d9470df4e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,6 +195,9 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE + - cat .aws/credentials + - cat .aws/config + - cat .aws/sso - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh From ab3679bef797c3b8a8a9d1a2596acb8e21485e69 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 11:33:17 -0500 Subject: [PATCH 038/222] sanity check --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d9470df4e..8ced2aada 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,9 +195,9 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - cat .aws/credentials - cat .aws/config - - cat .aws/sso + - cat .aws/sso + - cat .aws/credentials - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh From d080164a49ae689ca8f77ee6f8fcbb5da5620b44 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 11:35:54 -0500 Subject: [PATCH 039/222] echo aws config --- .gitlab/.gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8ced2aada..38e0615e5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,9 +195,7 @@ deploy ec2 instance: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - cat .aws/config - - cat .aws/sso - - cat .aws/credentials + - aws configure list - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh From 7c02e9c447fed82c02988dfd020f0b3cb1cbe682 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 12:37:15 -0500 Subject: [PATCH 040/222] added scp --- .gitlab/.gitlab-ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 38e0615e5..3c39fa448 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,8 +176,8 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -deploy ec2 instance: - stage: lint +deploy node for linux arm64: + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -188,14 +188,12 @@ deploy ec2 instance: AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" - INSTANCE_NAME: "Single_Node_Deployment" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - aws configure list - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p /root/.ssh @@ -205,6 +203,9 @@ deploy ec2 instance: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]' --profile $PROFILE + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Single_Node_Deployment}]' --profile $PROFILE - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - - echo $PUBLIC_IP \ No newline at end of file + - echo $PUBLIC_IP + - scp featurebase_linux_amd64 ec2-user@$PUBLIC_IP + dependencies: + - build for linux arm64 \ No newline at end of file From 86407ebcabeb012efb1f3c66796c111d877d8d7c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 12:39:11 -0500 Subject: [PATCH 041/222] sanity check for scp --- .gitlab/.gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3c39fa448..a65a9c0f8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -177,7 +177,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux arm64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -206,6 +206,5 @@ deploy node for linux arm64: - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Single_Node_Deployment}]' --profile $PROFILE - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - - scp featurebase_linux_amd64 ec2-user@$PUBLIC_IP - dependencies: - - build for linux arm64 \ No newline at end of file + - echo "scp test" > test.txt + - scp test.txt ec2-user@$PUBLIC_IP From 4f964aeef4f4c1a381fb17b9f9875689dc3163d6 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 12:53:39 -0500 Subject: [PATCH 042/222] updated ssh config --- .gitlab/.gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a65a9c0f8..6ea55f1d8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,8 +196,9 @@ deploy node for linux arm64: - aws configure set aws_profile $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - mkdir -p /root/.ssh - - chmod 700 /root/.ssh + - ssh-add <(echo "$SSH_PRIVATE_KEY") + - mkdir -p ~/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) @@ -207,4 +208,4 @@ deploy node for linux arm64: - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - echo "scp test" > test.txt - - scp test.txt ec2-user@$PUBLIC_IP + - scp -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP From 642d4fcb868aeca817fa942ccf3ff9f0dfef0da1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:27:03 -0500 Subject: [PATCH 043/222] fixed variable name --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6ea55f1d8..8b63d04f2 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,7 +196,7 @@ deploy node for linux arm64: - aws configure set aws_profile $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - ssh-add <(echo "$SSH_PRIVATE_KEY") + - ssh-add <(echo "$AWS_SSH_PRIVATE_KEY") - mkdir -p ~/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev From fae8d2d1163ebaf46378782c4ecccf4d458c3768 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:32:15 -0500 Subject: [PATCH 044/222] ssh add command --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8b63d04f2..69bce4706 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,7 +196,7 @@ deploy node for linux arm64: - aws configure set aws_profile $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - ssh-add <(echo "$AWS_SSH_PRIVATE_KEY") + - echo -n "$AWS_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - mkdir -p ~/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev From 4b2322160bee6a0d669939a617fee5193434f351 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:41:13 -0500 Subject: [PATCH 045/222] sanity --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 69bce4706..6685ccdcc 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,7 +196,7 @@ deploy node for linux arm64: - aws configure set aws_profile $PROFILE - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - echo -n "$AWS_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null + - echo "$AWS_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - - mkdir -p ~/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev From 98e5f6deaf6ad57c63f74b89cbe5904b308d28e9 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:45:11 -0500 Subject: [PATCH 046/222] ssh key --- .gitlab/server.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 .gitlab/server.sh diff --git a/.gitlab/server.sh b/.gitlab/server.sh new file mode 100755 index 000000000..5c7571b83 --- /dev/null +++ b/.gitlab/server.sh @@ -0,0 +1,5 @@ +#cloud-config +runcmd: + - sudo yum update + - sudo yum install postgresql14 + - sudo yum install golang1.16 \ No newline at end of file From cf365db944e7426b0f0a79b30d2a27361bf01905 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:47:47 -0500 Subject: [PATCH 047/222] changed ssh config --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6685ccdcc..60bd54c3f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -194,12 +194,12 @@ deploy node for linux arm64: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - echo "$AWS_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - + - echo gitlab-featurebase-dev | tr -d '\r' | ssh-add - - mkdir -p ~/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) From 7a5ba0e5d6a452572c3a2dc89ffde7ee22610db4 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:52:26 -0500 Subject: [PATCH 048/222] Revert "sanity check for scp" This reverts commit 86407ebcabeb012efb1f3c66796c111d877d8d7c. --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 60bd54c3f..876185e57 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -205,7 +205,8 @@ deploy node for linux arm64: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Single_Node_Deployment}]' --profile $PROFILE + - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - echo "scp test" > test.txt - - scp -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP From 590e5edc57b12740be233b901d0ffca88bde17cf Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 13:57:37 -0500 Subject: [PATCH 049/222] reverted to last working version --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 876185e57..04dbe2178 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -197,8 +197,8 @@ deploy node for linux arm64: - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - echo gitlab-featurebase-dev | tr -d '\r' | ssh-add - - mkdir -p ~/.ssh + - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 179a0e7624cb52fbe7d1c19f9f3fd349e4cdb74b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 14:03:14 -0500 Subject: [PATCH 050/222] fixed typo for copy path --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 04dbe2178..a3ba3f4e1 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -209,4 +209,4 @@ deploy node for linux arm64: - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - echo "scp test" > test.txt - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP:. From bf93278ca30c370e5e9630b6d4e76be7db8b6e37 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 14:24:02 -0500 Subject: [PATCH 051/222] scp binary --- .gitlab/.gitlab-ci.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a3ba3f4e1..beedce17b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,8 +176,8 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -deploy node for linux arm64: - stage: lint +deploy node for linux amd64: + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -204,9 +204,16 @@ deploy node for linux arm64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Single_Node_Deployment}]' --profile $PROFILE + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE - sleep 30 - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=Single_Node_Deployment' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - - echo "scp test" > test.txt - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev test.txt ec2-user@$PUBLIC_IP:. + - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) + - echo $INSTANCE_ID > linux-amd64-instance.txt + - echo $INSTANCE_ID + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + dependencies: + jobs: build for linux amd64 + artifacts: + paths: + - linux-amd64-instance.txt \ No newline at end of file From 6531f907472cd02399a0de62a82a37792111642b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 14:26:20 -0500 Subject: [PATCH 052/222] fixed syntax error --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index beedce17b..5621f38cc 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -212,8 +212,8 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - dependencies: - jobs: build for linux amd64 + needs: + job: build for linux amd64 artifacts: paths: - linux-amd64-instance.txt \ No newline at end of file From 103312be8f9fd53a268788f8672d50e643f674a5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 5 Nov 2021 15:04:09 -0500 Subject: [PATCH 053/222] added cloutinit --- .gitlab/.gitlab-ci.yml | 6 +++--- .gitlab/server.sh | 5 ----- .gitlab/server.yml | 9 ++++++--- 3 files changed, 9 insertions(+), 11 deletions(-) delete mode 100755 .gitlab/server.sh diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 5621f38cc..29a6e5b64 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -109,7 +109,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: build + stage: lint image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 extends: .go-cache variables: @@ -177,7 +177,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux amd64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -204,7 +204,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data-file server.yml - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP diff --git a/.gitlab/server.sh b/.gitlab/server.sh deleted file mode 100755 index 5c7571b83..000000000 --- a/.gitlab/server.sh +++ /dev/null @@ -1,5 +0,0 @@ -#cloud-config -runcmd: - - sudo yum update - - sudo yum install postgresql14 - - sudo yum install golang1.16 \ No newline at end of file diff --git a/.gitlab/server.yml b/.gitlab/server.yml index 5c7571b83..3fe5243ff 100755 --- a/.gitlab/server.yml +++ b/.gitlab/server.yml @@ -1,5 +1,8 @@ #cloud-config +repo_update: true +repo_upgrade: all + runcmd: - - sudo yum update - - sudo yum install postgresql14 - - sudo yum install golang1.16 \ No newline at end of file + - yum update + - yum install postgresql + - yum install golang \ No newline at end of file From 78713c6a8c900f0efc3c2caa5ad6042e77e8ce11 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 08:39:43 -0600 Subject: [PATCH 054/222] test cloudinit --- .gitlab/cloud-init.sh | 24 ++++++++++++++++++++++++ .gitlab/server.yml | 8 -------- 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100755 .gitlab/cloud-init.sh delete mode 100755 .gitlab/server.yml diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh new file mode 100755 index 000000000..3ef2133ff --- /dev/null +++ b/.gitlab/cloud-init.sh @@ -0,0 +1,24 @@ +#!/bin/bash -ex +# generate log +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + +# Install packages +yum update -y +yum install golang -y # latest verion in ec2 is 1.15.14 +yum install postgresql -y + +# Configure host system +# N=200000 +# sysctl -w vm.max_map_count=N +# echo N > /proc/sys/vm/max_map_count +ulimit -n -S 200000 + +# # install go 1.16.9 +# curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz +# tar xvf go1.16.9.linux-amd64.tar.gz +# chown -R root:root ./go +# mv go /usr/local +# echo 'export GOPATH=$HOME/work' >> ~/.profile +# echo 'export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin' >> ~/.profile +# chown ec2-user ~/.profile +# source ~/.profile diff --git a/.gitlab/server.yml b/.gitlab/server.yml deleted file mode 100755 index 3fe5243ff..000000000 --- a/.gitlab/server.yml +++ /dev/null @@ -1,8 +0,0 @@ -#cloud-config -repo_update: true -repo_upgrade: all - -runcmd: - - yum update - - yum install postgresql - - yum install golang \ No newline at end of file From 606b7184289b564322408e43fc06ac067ff1a0b0 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 08:42:40 -0600 Subject: [PATCH 055/222] fixed path for cloud-init --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 29a6e5b64..92a82b0f3 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -204,7 +204,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data-file server.yml + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://featurebase/.gitlab/cloud-init.sh - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP From 0e564a00e8b4d25163feb0de9bdef043c3ca538d Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 08:47:10 -0600 Subject: [PATCH 056/222] path for cloud-init --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 92a82b0f3..93714a717 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -204,7 +204,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://featurebase/.gitlab/cloud-init.sh + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://cloud-init.sh - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP From 57d4eb45ded68ef704c1a6479e90342be5baf698 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 08:53:01 -0600 Subject: [PATCH 057/222] gitlab path test2 --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 93714a717..26a786640 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -204,7 +204,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://cloud-init.sh + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh - sleep 30 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP From dfc98e3000a7e642e18c76d640946eb61f4b7484 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:11:52 -0600 Subject: [PATCH 058/222] updated sleep time --- .gitlab/.gitlab-ci.yml | 2 +- .gitlab/cloud-init.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 26a786640..47b571d2e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -205,7 +205,7 @@ deploy node for linux amd64: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh - - sleep 30 + - sleep 120 - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index 3ef2133ff..e192938ba 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -8,10 +8,10 @@ yum install golang -y # latest verion in ec2 is 1.15.14 yum install postgresql -y # Configure host system -# N=200000 -# sysctl -w vm.max_map_count=N -# echo N > /proc/sys/vm/max_map_count -ulimit -n -S 200000 +cat /proc/sys/fs/file-max +sysctl -w fs.file-max=262144 +sysctl -p +cat /proc/sys/fs/file-max # # install go 1.16.9 # curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz From f0b2e6beeaba2d074161cce23fe707ff40ca7b78 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:17:02 -0600 Subject: [PATCH 059/222] updated permissions for ssh key --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 47b571d2e..b9ef214ff 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,6 +195,7 @@ deploy node for linux amd64: - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev + - chmod 400 gitlab-featurebase-dev - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p ~/.ssh From 46d6ced5b46251095a9297909da95cb6ef7fca57 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:24:40 -0600 Subject: [PATCH 060/222] updated ssh permissions --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index b9ef214ff..efb828c44 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,6 +200,7 @@ deploy node for linux amd64: - eval `ssh-agent -s` - mkdir -p ~/.ssh - chmod 700 /root/.ssh + - chmod 600 /root/.ssh/authorized_keys - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) From 76ba07e806e17f9bf5d4fce10a731414c69f29f9 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:29:46 -0600 Subject: [PATCH 061/222] update ssh key filename --- .gitlab/.gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index efb828c44..bdd73b7d5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -194,13 +194,12 @@ deploy node for linux amd64: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - chmod 400 gitlab-featurebase-dev - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p ~/.ssh - chmod 700 /root/.ssh - - chmod 600 /root/.ssh/authorized_keys - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) @@ -213,7 +212,7 @@ deploy node for linux amd64: - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 artifacts: From df99dd7578c92bcabdba1887eb6ad8e6543cac0a Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Sun, 7 Nov 2021 09:31:34 -0600 Subject: [PATCH 062/222] fixed typo --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bdd73b7d5..576eb8e7a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -195,7 +195,7 @@ deploy node for linux amd64: - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - - chmod 400 gitlab-featurebase-dev + - chmod 400 gitlab-featurebase-dev.pem - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p ~/.ssh From 5b5c003b1ed666226a1086d0b1f1e6e9e2f749d5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 07:43:31 -0600 Subject: [PATCH 063/222] private key fix --- .gitlab/.gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 576eb8e7a..903cfb53f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -199,6 +199,7 @@ deploy node for linux amd64: - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - mkdir -p ~/.ssh + - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: @@ -212,7 +213,7 @@ deploy node for linux amd64: - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 artifacts: From 79d1a005ce6f0873e45c6e13161d3dab9a75d629 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 07:55:42 -0600 Subject: [PATCH 064/222] job to terminate ec2 instance --- .gitlab/.gitlab-ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 903cfb53f..e8a320a7b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -214,8 +214,21 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - sleep 120 needs: job: build for linux amd64 artifacts: paths: - - linux-amd64-instance.txt \ No newline at end of file + - linux-amd64-instance.txt + +terminate node for linux amd64: + stage: lint + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + INSTANCE_ID + before_script: + - INSTANCE_ID=$(cat linux-amd64-instance.txt) + script: + - aws ec2 terminate-instances --instance-ids $INSTANCE_ID + needs: + job: deploy node for linux amd64 From 4c2832ab31e37f0303a6a3c04793f8dbf2b34d9c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 07:59:15 -0600 Subject: [PATCH 065/222] fixed yaml syntax error --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e8a320a7b..6d698f258 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -225,7 +225,7 @@ terminate node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: - INSTANCE_ID + INSTANCE_ID: "" before_script: - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: From 1befe2ad320a833ecf4e818d8ab1ccc2c902c76c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 08:10:20 -0600 Subject: [PATCH 066/222] added aws config for terminate job --- .gitlab/.gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6d698f258..a3599bbe7 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -227,6 +227,9 @@ terminate node for linux amd64: variables: INSTANCE_ID: "" before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID From aae537424851ad244c0eb7aff02d92f6d92fba52 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 08:18:37 -0600 Subject: [PATCH 067/222] fixed stages --- .gitlab/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a3599bbe7..bfb881491 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -109,7 +109,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: lint + stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 extends: .go-cache variables: @@ -177,7 +177,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -222,7 +222,7 @@ deploy node for linux amd64: - linux-amd64-instance.txt terminate node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" From e629e0165de388ab2fb44cc8032f42487d98105d Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 08:42:49 -0600 Subject: [PATCH 068/222] install correct version of go --- .gitlab/.gitlab-ci.yml | 6 +++--- .gitlab/cloud-init.sh | 22 ++++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bfb881491..a3599bbe7 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -109,7 +109,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: build + stage: lint image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 extends: .go-cache variables: @@ -177,7 +177,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux amd64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -222,7 +222,7 @@ deploy node for linux amd64: - linux-amd64-instance.txt terminate node for linux amd64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index e192938ba..18257a90b 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -4,21 +4,19 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 # Install packages yum update -y -yum install golang -y # latest verion in ec2 is 1.15.14 +# yum install golang -y # latest verion in ec2 is 1.15.14 yum install postgresql -y # Configure host system -cat /proc/sys/fs/file-max +echo 'cat /proc/sys/fs/file-max' sysctl -w fs.file-max=262144 sysctl -p -cat /proc/sys/fs/file-max +echo 'cat /proc/sys/fs/file-max' -# # install go 1.16.9 -# curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz -# tar xvf go1.16.9.linux-amd64.tar.gz -# chown -R root:root ./go -# mv go /usr/local -# echo 'export GOPATH=$HOME/work' >> ~/.profile -# echo 'export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin' >> ~/.profile -# chown ec2-user ~/.profile -# source ~/.profile +# install go 1.16.9 +curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz +tar xvf go1.16.9.linux-amd64.tar.gz +chown -R root:root ./go +mv go /usr/local +echo "export PATH=$PATH:/usr/local/go/bin" | sudo tee -a /etc/profile > /dev/null +source /etc/profile From ebde0142da01c1c4d205a7b363c2a782c638fd42 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 09:14:47 -0600 Subject: [PATCH 069/222] add stage to configure featurebase --- .gitlab/.gitlab-ci.yml | 29 ++++++++++++++++++++--------- .gitlab/cloud-init.sh | 4 ++-- .gitlab/configureFeatureBase.json | 20 ++++++++++++++++++++ .gitlab/featurebase.conf | 30 ++++++++++++++++++++++++++++++ .gitlab/featurebase.service | 13 +++++++++++++ 5 files changed, 85 insertions(+), 11 deletions(-) create mode 100644 .gitlab/configureFeatureBase.json create mode 100644 .gitlab/featurebase.conf create mode 100644 .gitlab/featurebase.service diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a3599bbe7..bf210f5a9 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,6 +176,11 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" +before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -185,14 +190,9 @@ deploy node for linux amd64: SECURITY_GROUP: "" SUBNET_ID: "" PUBLIC_IP: "" - AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY PROFILE: "default" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - chmod 400 gitlab-featurebase-dev.pem @@ -214,6 +214,7 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - sleep 120 needs: job: build for linux amd64 @@ -221,17 +222,27 @@ deploy node for linux amd64: paths: - linux-amd64-instance.txt +run featurebase in linux amd64 node: + stage: lint + image: + variables: + INSTANCE_ID: "" + before_script: + - INSTANCE_ID=$(cat linux-amd64-instance.txt) + script: + - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceIds=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json + needs: + job: + deploy node for linux amd64 + terminate node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: deploy node for linux amd64 + job: run featurebase in linux amd64 node diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index 18257a90b..f9f289582 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -4,7 +4,6 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 # Install packages yum update -y -# yum install golang -y # latest verion in ec2 is 1.15.14 yum install postgresql -y # Configure host system @@ -13,7 +12,8 @@ sysctl -w fs.file-max=262144 sysctl -p echo 'cat /proc/sys/fs/file-max' -# install go 1.16.9 +# yum install golang -y # latest verion in ec2 is 1.15.14 +# install go 1.16.9 manually curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz tar xvf go1.16.9.linux-amd64.tar.gz chown -R root:root ./go diff --git a/.gitlab/configureFeatureBase.json b/.gitlab/configureFeatureBase.json new file mode 100644 index 000000000..569b6d0bd --- /dev/null +++ b/.gitlab/configureFeatureBase.json @@ -0,0 +1,20 @@ +{ + "Parameters": { + "commands": [ + "#!/bin/bash", + "mv featurebase_linux_amd64 /usr/local/bin/", + "mv featurebase.conf /etc/", + "mv featurebase.service /etc/systemd/system/", + "adduser molecula", + "sudo mkdir /var/log/molecula", + "sudo chown molecula /var/log/molecula", + "sudo mkdir -p /opt/molecula/featurebase", + "sudo chown molecula /opt/molecula/featurebase", + "systemctl daemon-reload", + "sudo systemctl start featurebase", + "sudo systemctl enable featurebase", + "sudo systemctl status featurebase", + "curl localhost:10101" + ] + } +} \ No newline at end of file diff --git a/.gitlab/featurebase.conf b/.gitlab/featurebase.conf new file mode 100644 index 000000000..7716d0f6b --- /dev/null +++ b/.gitlab/featurebase.conf @@ -0,0 +1,30 @@ +name = "pilosa1" +bind = "0.0.0.0:10101" +bind-grpc = "0.0.0.0:20101" + +data-dir = "/opt/molecula/featurebase" +log-path = "/var/log/molecula/featurebase.log" + +max-file-count=900000 +max-map-count=900000 + +long-query-time = "10s" + +[postgres] + + bind = "localhost:55432" + +[cluster] + + name = "cluster1" + replicas = 1 + +[etcd] + + listen-client-address = "http://localhost:10401" + listen-peer-address = "http://localhost:10301" + initial-cluster = "pilosa1=http://localhost:10301" + +[metric] + + service = "prometheus" \ No newline at end of file diff --git a/.gitlab/featurebase.service b/.gitlab/featurebase.service new file mode 100644 index 000000000..a543b92af --- /dev/null +++ b/.gitlab/featurebase.service @@ -0,0 +1,13 @@ +# Not Ansible managed + +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] \ No newline at end of file From 27e39b933fdcf7ff5f2b6510a02b60f8edebdc23 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 09:21:14 -0600 Subject: [PATCH 070/222] reverted change to aws configure --- .gitlab/.gitlab-ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bf210f5a9..02be1f147 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,11 +176,6 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -193,6 +188,9 @@ deploy node for linux amd64: PROFILE: "default" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - chmod 400 gitlab-featurebase-dev.pem @@ -228,6 +226,9 @@ run featurebase in linux amd64 node: variables: INSTANCE_ID: "" before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceIds=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json @@ -241,6 +242,9 @@ terminate node for linux amd64: variables: INSTANCE_ID: "" before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID From 1cdc8e495921c158fbadb273033ac353d0c82fce Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 09:33:34 -0600 Subject: [PATCH 071/222] added aws image --- .gitlab/.gitlab-ci.yml | 4 ++-- .gitlab/cloud-init.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 02be1f147..6e941512d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -213,7 +213,6 @@ deploy node for linux amd64: - echo $INSTANCE_ID - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - - sleep 120 needs: job: build for linux amd64 artifacts: @@ -222,7 +221,7 @@ deploy node for linux amd64: run featurebase in linux amd64 node: stage: lint - image: + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" before_script: @@ -232,6 +231,7 @@ run featurebase in linux amd64 node: - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceIds=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json + - sleep 120 needs: job: deploy node for linux amd64 diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index f9f289582..e87c6124a 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -18,5 +18,5 @@ curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz tar xvf go1.16.9.linux-amd64.tar.gz chown -R root:root ./go mv go /usr/local -echo "export PATH=$PATH:/usr/local/go/bin" | sudo tee -a /etc/profile > /dev/null +echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null source /etc/profile From 38a17e2b941c3f0694096c5f6453cc3d28ba1346 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 09:40:14 -0600 Subject: [PATCH 072/222] fixed typo --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6e941512d..f614e052f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -230,7 +230,7 @@ run featurebase in linux amd64 node: - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceIds=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json + - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceId=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 needs: job: From 14265090db186114637a0814e9abd195ed8d75fd Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 10:20:10 -0600 Subject: [PATCH 073/222] fixed arguments --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f614e052f..9db55187a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -230,7 +230,7 @@ run featurebase in linux amd64 node: - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - aws ssm send-command --document-name "AWS-RunShellScript" --targets InstanceId=$INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json + - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values={$INSTANCE_ID}" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 needs: job: From d3cc4d7f2a8c67d918b0a3f9c11f26cb39cd2b7f Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 10:26:59 -0600 Subject: [PATCH 074/222] added argument to scp --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 9db55187a..fb71ade61 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -211,8 +211,8 @@ deploy node for linux amd64: - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - - scp -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - - scp -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 artifacts: From 22c081528833aef34b259a230831eaa581e79372 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 10:31:57 -0600 Subject: [PATCH 075/222] fixed syntax error --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index fb71ade61..b018ef816 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -230,7 +230,7 @@ run featurebase in linux amd64 node: - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values={$INSTANCE_ID}" --cli-input-json file://.gitlab/configureFeatureBase.json + - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 needs: job: From 4a42571dd9cb81ecdb823503fb1e14ca57aaeea0 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 10:45:10 -0600 Subject: [PATCH 076/222] fix for go --- .gitlab/.gitlab-ci.yml | 1 + .gitlab/cloud-init.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index b018ef816..d6b57420b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -229,6 +229,7 @@ run featurebase in linux amd64 node: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) + - echo $INSTANCE_ID script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index e87c6124a..c86818e48 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -18,5 +18,5 @@ curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz tar xvf go1.16.9.linux-amd64.tar.gz chown -R root:root ./go mv go /usr/local -echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null -source /etc/profile +echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile.d > /dev/null +source /etc/profile.d From 8edfc0c9ff81d74311a51a9f21bc78b16454ed85 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 11:00:49 -0600 Subject: [PATCH 077/222] made instanceId a global variable --- .gitlab/.gitlab-ci.yml | 3 +-- .gitlab/cloud-init.sh | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d6b57420b..8545a6f2f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,6 +12,7 @@ include: - .go/pkg/mod/ variables: GOVERSION: "1.16.9" + INSTANCE_ID: "" stages: - lint @@ -228,8 +229,6 @@ run featurebase in linux amd64 node: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - INSTANCE_ID=$(cat linux-amd64-instance.txt) - - echo $INSTANCE_ID script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index c86818e48..e87c6124a 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -18,5 +18,5 @@ curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz tar xvf go1.16.9.linux-amd64.tar.gz chown -R root:root ./go mv go /usr/local -echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile.d > /dev/null -source /etc/profile.d +echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null +source /etc/profile From 26d77c51b53c0a4a2d172f6eeb80ccab2c52fc51 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 11:06:55 -0600 Subject: [PATCH 078/222] removed variable overwrite --- .gitlab/.gitlab-ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8545a6f2f..e1cbaef21 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -223,12 +223,11 @@ deploy node for linux amd64: run featurebase in linux amd64 node: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - INSTANCE_ID: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" + - echo $INSTANCE_ID script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json - sleep 120 @@ -239,13 +238,10 @@ run featurebase in linux amd64 node: terminate node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - INSTANCE_ID: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: From 2b315c6634eade3cf08b1987536239231ed99bd1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 11:23:05 -0600 Subject: [PATCH 079/222] removed instance id file --- .gitlab/.gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e1cbaef21..c590c1885 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -210,15 +210,11 @@ deploy node for linux amd64: - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - - echo $INSTANCE_ID > linux-amd64-instance.txt - echo $INSTANCE_ID - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 - artifacts: - paths: - - linux-amd64-instance.txt run featurebase in linux amd64 node: stage: lint @@ -242,6 +238,7 @@ terminate node for linux amd64: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" + - echo $INSTANCE_ID script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: From 6d4ae4c345f4b65620a7db6dde2b27a355dc7f52 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 11:34:10 -0600 Subject: [PATCH 080/222] changed var assignment --- .gitlab/.gitlab-ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c590c1885..c1675d40b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -210,19 +210,25 @@ deploy node for linux amd64: - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - - echo $INSTANCE_ID + - echo $INSTANCE_ID > linux-amd64-instance.txt - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. needs: job: build for linux amd64 + artifacts: + paths: + - linux-amd64-instance.txt run featurebase in linux amd64 node: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + INSTANCE_ID: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" + - INSTANCE_ID=$(cat linux-amd64-instance.txt) - echo $INSTANCE_ID script: - aws ssm send-command --document-name "AWS-RunShellScript" --targets "Key=InstanceIds,Values=$INSTANCE_ID" --cli-input-json file://.gitlab/configureFeatureBase.json @@ -234,10 +240,13 @@ run featurebase in linux amd64 node: terminate node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + INSTANCE_ID: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" + - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 11:39:17 -0600 Subject: [PATCH 081/222] updated var assignment --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c1675d40b..457aa0b4d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -228,7 +228,7 @@ run featurebase in linux amd64 node: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - INSTANCE_ID=$(cat linux-amd64-instance.txt) + - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 11:45:41 -0600 Subject: [PATCH 082/222] access array value --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 457aa0b4d..dba79b098 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -231,7 +231,7 @@ run featurebase in linux amd64 node: - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 11:55:17 -0600 Subject: [PATCH 083/222] fix var assignment --- .gitlab/.gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index dba79b098..f238ff88f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -229,9 +229,8 @@ run featurebase in linux amd64 node: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 12:00:17 -0600 Subject: [PATCH 084/222] removed targets flag --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f238ff88f..a7b229c89 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -230,7 +230,7 @@ run featurebase in linux amd64 node: - aws configure set region "us-east-2" - INSTANCE_ID=$(echo "$( Date: Mon, 8 Nov 2021 13:21:07 -0600 Subject: [PATCH 085/222] added iam role --- .gitlab/.gitlab-ci.yml | 8 +++++--- .gitlab/cloud-init.sh | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a7b229c89..3540c1a8c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -213,6 +213,7 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. + - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm needs: job: build for linux amd64 artifacts: @@ -228,9 +229,10 @@ run featurebase in linux amd64 node: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - INSTANCE_ID=$(echo "$( /dev/null source /etc/profile + +# install aws session manager pluggin +curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" +yum install -y session-manager-plugin.rpm \ No newline at end of file From 3d11f9cebf43bfd13883b7ec967e88aa7246af5b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 13:35:20 -0600 Subject: [PATCH 086/222] added profile --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3540c1a8c..8c1d2d0e4 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -225,6 +225,7 @@ run featurebase in linux amd64 node: image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" + PROFILE: "default" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY From 48295a5491be764dad3b9f7043cf15870f2218f5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 13:50:47 -0600 Subject: [PATCH 087/222] added abs paths and add sleep time --- .gitlab/.gitlab-ci.yml | 1 + .gitlab/configureFeatureBase.json | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8c1d2d0e4..bed04357b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -214,6 +214,7 @@ deploy node for linux amd64: - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm + - sleep 120 needs: job: build for linux amd64 artifacts: diff --git a/.gitlab/configureFeatureBase.json b/.gitlab/configureFeatureBase.json index 569b6d0bd..6afd135d9 100644 --- a/.gitlab/configureFeatureBase.json +++ b/.gitlab/configureFeatureBase.json @@ -2,9 +2,9 @@ "Parameters": { "commands": [ "#!/bin/bash", - "mv featurebase_linux_amd64 /usr/local/bin/", - "mv featurebase.conf /etc/", - "mv featurebase.service /etc/systemd/system/", + "mv /home/ec2-user/featurebase_linux_amd64 /usr/local/bin/featurebase", + "mv /home/ec2-user/featurebase.conf /etc/", + "mv /home/ec2-user/featurebase.service /etc/systemd/system/", "adduser molecula", "sudo mkdir /var/log/molecula", "sudo chown molecula /var/log/molecula", From af4b420d661cfa7cd413c314e5c3d08d524b62d1 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 13:58:34 -0600 Subject: [PATCH 088/222] added region --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bed04357b..ae18e9f1d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -234,7 +234,7 @@ run featurebase in linux amd64 node: - aws configure set aws_profile $PROFILE - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE + - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: job: From d92bd37d71a87d8739932c640084508f98971939 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:20:04 -0600 Subject: [PATCH 089/222] added another sleep --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ae18e9f1d..b19fe0757 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -234,6 +234,7 @@ run featurebase in linux amd64 node: - aws configure set aws_profile $PROFILE - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: + - sleep 60 - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: From 1b503a3ddfb1573c941c19d095f94860eb2b7e0c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:32:00 -0600 Subject: [PATCH 090/222] made the sleep time longer --- .gitlab/.gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index b19fe0757..d28d3c5ce 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -214,7 +214,6 @@ deploy node for linux amd64: - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm - - sleep 120 needs: job: build for linux amd64 artifacts: @@ -234,7 +233,7 @@ run featurebase in linux amd64 node: - aws configure set aws_profile $PROFILE - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - sleep 60 + - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: From aab7a2e4be2627ac037d8ba8b01d735c7e243fb4 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:33:34 -0600 Subject: [PATCH 091/222] fixed stages --- .gitlab/.gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d28d3c5ce..e04dab61d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -110,7 +110,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: lint + stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 extends: .go-cache variables: @@ -178,7 +178,7 @@ build container fb: - echo Created docker featurebase image with tag "$tag" deploy node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" @@ -237,11 +237,10 @@ run featurebase in linux amd64 node: - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: - job: - deploy node for linux amd64 + job: deploy node for linux amd64 terminate node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" @@ -254,4 +253,5 @@ terminate node for linux amd64: script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: + job: deploy node for linux amd64 job: run featurebase in linux amd64 node From 4c291180b979aa638688b2defc535ae19adc5d14 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:38:35 -0600 Subject: [PATCH 092/222] added artifact paths --- .gitlab/.gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index e04dab61d..57c0c98a0 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -238,6 +238,9 @@ run featurebase in linux amd64 node: - sleep 120 needs: job: deploy node for linux amd64 + artifacts: + paths: + - linux-amd64-instance.txt terminate node for linux amd64: stage: integration @@ -253,5 +256,4 @@ terminate node for linux amd64: script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: deploy node for linux amd64 job: run featurebase in linux amd64 node From d5f5340016e576ae5646780054379959cbd06f51 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:44:33 -0600 Subject: [PATCH 093/222] fixed syntax error --- .gitlab/.gitlab-ci.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 57c0c98a0..16c5f4a11 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,7 +12,6 @@ include: - .go/pkg/mod/ variables: GOVERSION: "1.16.9" - INSTANCE_ID: "" stages: - lint @@ -215,7 +214,7 @@ deploy node for linux amd64: - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm needs: - job: build for linux amd64 + - job: build for linux amd64 artifacts: paths: - linux-amd64-instance.txt @@ -237,10 +236,7 @@ run featurebase in linux amd64 node: - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 needs: - job: deploy node for linux amd64 - artifacts: - paths: - - linux-amd64-instance.txt + - job: deploy node for linux amd64 terminate node for linux amd64: stage: integration @@ -256,4 +252,5 @@ terminate node for linux amd64: script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: run featurebase in linux amd64 node + - job: deploy node for linux amd64 + - job: run featurebase in linux amd64 node From 91e93cb975eaceb4a00a67573b0c7e8fa6193b8b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 14:47:14 -0600 Subject: [PATCH 094/222] fixed stage --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 16c5f4a11..66e266a79 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -220,7 +220,7 @@ deploy node for linux amd64: - linux-amd64-instance.txt run featurebase in linux amd64 node: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: INSTANCE_ID: "" From d48da50cf690686cfdbb486c9bd4b858a2578caa Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 15:15:26 -0600 Subject: [PATCH 095/222] consolidated some commands --- .gitlab/.gitlab-ci.yml | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 66e266a79..f34d6d93b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -176,6 +176,7 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" +# deploy EC2 instance and upload featurebase binary deploy node for linux amd64: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -205,7 +206,7 @@ deploy node for linux amd64: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh - - sleep 120 + - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) @@ -213,31 +214,15 @@ deploy node for linux amd64: - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm - needs: + - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called + - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 + - sleep 120 + needs: - job: build for linux amd64 artifacts: paths: - linux-amd64-instance.txt -run featurebase in linux amd64 node: - stage: integration - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - INSTANCE_ID: "" - PROFILE: "default" - before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - - aws configure set aws_profile $PROFILE - - INSTANCE_ID=$(cat linux-amd64-instance.txt) - script: - - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called - - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - - sleep 120 - needs: - - job: deploy node for linux amd64 - terminate node for linux amd64: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -253,4 +238,3 @@ terminate node for linux amd64: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: deploy node for linux amd64 - - job: run featurebase in linux amd64 node From 1bda8e439d0811a3c7a3ab5b4d147dc7a00227ce Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 15:26:01 -0600 Subject: [PATCH 096/222] fixed indentation --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 373993c79..8f317eed1 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -232,7 +232,7 @@ deploy node for linux amd64: - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - sleep 120 - needs: + needs: - job: build for linux amd64 artifacts: paths: From 05be45194a0ff106024a70dd35718d1d9e03eb3c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 15:54:12 -0600 Subject: [PATCH 097/222] combined iam role with run-instance --- .gitlab/.gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 8f317eed1..904f6c5cf 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -220,7 +220,7 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP @@ -228,10 +228,7 @@ deploy node for linux amd64: - echo $INSTANCE_ID > linux-amd64-instance.txt - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - - aws ec2 associate-iam-instance-profile --instance-id $INSTANCE_ID --iam-instance-profile Name=featurebase-dev-ssm - - sleep 240 # needed to allow enough time for the ssm to take into effect before the next command can get called - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - - sleep 120 needs: - job: build for linux amd64 artifacts: From a8707b55bddbbd09aab956834604061ed39d3fb2 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 8 Nov 2021 16:14:25 -0600 Subject: [PATCH 098/222] updated instance type --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 904f6c5cf..247e377bd 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -197,7 +197,7 @@ deploy node for linux amd64: image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: AMI: "" - INSTANCE: "t2.micro" + INSTANCE: "t3a.large" SECURITY_GROUP: "" SUBNET_ID: "" PUBLIC_IP: "" From 9ef057cc85e8767994aa657c4b269e8ddcf1c44e Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 10:27:31 -0600 Subject: [PATCH 099/222] fix typo --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 247e377bd..544b9f9e4 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -218,7 +218,7 @@ deploy node for linux amd64: - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data From 02baec2c6e258ba21d7a64d48d22b49dfef0ae9e Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 11:02:51 -0600 Subject: [PATCH 100/222] oops --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 544b9f9e4..ee4ec0bd2 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -218,7 +218,7 @@ deploy node for linux amd64: - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data From 253ca1182e3ba88009c7f782587f764c39c4b345 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 11 Nov 2021 15:46:45 -0600 Subject: [PATCH 101/222] upgrade go --- .circleci/config.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 30b2ef5e4..65534faa6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -263,7 +263,7 @@ workflows: - setup matrix: parameters: - golang_version: ["1.14.15", "1.15.8", "1.16.3"] + golang_version: ["1.15.8", "1.16.10"] - test: name: << matrix.test_make_target >> resource_class: xlarge diff --git a/Makefile b/Makefile index 9878b09d8..e2f659cf2 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) LDFLAGS="-X github.com/molecula/featurebase/v2.Version=$(VERSION) -X github.com/molecula/featurebase/v2.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v2.Variant=$(VARIANT) -X github.com/molecula/featurebase/v2.Commit=$(COMMIT) -X github.com/molecula/featurebase/v2.TrialDeadline=$(TRIAL_DEADLINE)" -GO_VERSION=1.16.3 +GO_VERSION=1.16.10 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia From 531957c51f228f923bc184858aa6c8e6e60fe8ed Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 16:05:43 -0600 Subject: [PATCH 102/222] hope --- .gitlab/.gitlab-ci.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ee4ec0bd2..c04d3b02f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -36,14 +36,6 @@ golangci-lint: - echo "Checking for issues in new code" - golangci-lint run -v -install lattice: - stage: test - image: node:14 - script: - - echo "Installing lattice dependencies..." - - cd lattice - - yarn install - build lattice: stage: test image: node:14 @@ -52,13 +44,12 @@ build lattice: script: - echo "Building lattice..." - cd lattice + - yarn install - yarn build - tar -czvf lattice.tar.gz lattice/build artifacts: paths: - lattice/lattice.tar.gz - needs: - - job: install lattice allow_failure: true run jest tests: From d449c8449fbf76b486c1ef34d77ac9d05997a511 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 16:17:38 -0600 Subject: [PATCH 103/222] don't treat warnings as errors --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c04d3b02f..f6c6c3cd2 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -40,7 +40,7 @@ build lattice: stage: test image: node:14 variables: - CI: "true" + CI: "false" script: - echo "Building lattice..." - cd lattice From 58c01793cb1d83311a312226fb5a0b092e1a5223 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 16:25:51 -0600 Subject: [PATCH 104/222] this ought to do it --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f6c6c3cd2..286c46da2 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -46,11 +46,11 @@ build lattice: - cd lattice - yarn install - yarn build + - cd ../ - tar -czvf lattice.tar.gz lattice/build artifacts: paths: - - lattice/lattice.tar.gz - allow_failure: true + - lattice.tar.gz run jest tests: stage: test From bd209b3005e86cd805aa11b026989bcfa3e977fc Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 16:46:10 -0600 Subject: [PATCH 105/222] please god work --- .gitlab/.gitlab-ci.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 286c46da2..15b154c5b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -101,13 +101,16 @@ upload to sonarcloud: build for linux amd64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + image: golang:1.16.10 extends: .go-cache variables: GOOS: "linux" GOARCH: "amd64" script: - - go build -o featurebase_linux_amd64 ./cmd/featurebase + - go get github.com/rakyll/statik + - tar -xzvf lattice.tar.gz + - /go/bin/statik -src=./lattice -dest=. + - make build FLAGS="-o featurebase_linux_amd64" artifacts: paths: - featurebase_linux_amd64 @@ -120,7 +123,10 @@ build for linux arm64: GOOS: "linux" GOARCH: "arm64" script: - - go build -o featurebase_linux_arm64 ./cmd/featurebase + - go get github.com/rakyll/statik + - tar -xzvf lattice.tar.gz + - /go/bin/statik -src=./lattice -dest=. + - make build FLAGS="-o featurebase_linux_arm64" artifacts: paths: - featurebase_linux_arm64 @@ -133,7 +139,10 @@ build for darwin amd64: GOOS: "darwin" GOARCH: "amd64" script: - - go build -o featurebase_darwin_amd64 ./cmd/featurebase + - go get github.com/rakyll/statik + - tar -xzvf lattice.tar.gz + - /go/bin/statik -src=./lattice -dest=. + - make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: - featurebase_darwin_amd64 @@ -146,7 +155,10 @@ build for darwin arm64: GOOS: "darwin" GOARCH: "arm64" script: - - go build -o featurebase_darwin_arm64 ./cmd/featurebase + - go get github.com/rakyll/statik + - tar -xzvf lattice.tar.gz + - /go/bin/statik -src=./lattice -dest=. + - make build FLAGS="-o featurebase_darwin_arm64" artifacts: paths: - featurebase_darwin_arm64 From 37866204d74f6c04c45c8e85a22901b30bc8f6ba Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 17:09:14 -0600 Subject: [PATCH 106/222] use statik right --- .gitlab/.gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 15b154c5b..d3c468df5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -109,7 +109,7 @@ build for linux amd64: script: - go get github.com/rakyll/statik - tar -xzvf lattice.tar.gz - - /go/bin/statik -src=./lattice -dest=. + - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_linux_amd64" artifacts: paths: @@ -125,7 +125,7 @@ build for linux arm64: script: - go get github.com/rakyll/statik - tar -xzvf lattice.tar.gz - - /go/bin/statik -src=./lattice -dest=. + - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_linux_arm64" artifacts: paths: @@ -141,7 +141,7 @@ build for darwin amd64: script: - go get github.com/rakyll/statik - tar -xzvf lattice.tar.gz - - /go/bin/statik -src=./lattice -dest=. + - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: @@ -157,7 +157,7 @@ build for darwin arm64: script: - go get github.com/rakyll/statik - tar -xzvf lattice.tar.gz - - /go/bin/statik -src=./lattice -dest=. + - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_arm64" artifacts: paths: From 0ab58b9132d4c834b1a555fc26a38d4a1461c41a Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 17:23:34 -0600 Subject: [PATCH 107/222] untar properly --- .gitlab/.gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d3c468df5..462977fa6 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -108,7 +108,7 @@ build for linux amd64: GOARCH: "amd64" script: - go get github.com/rakyll/statik - - tar -xzvf lattice.tar.gz + - tar -xvf lattice.tar.gz - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_linux_amd64" artifacts: @@ -124,7 +124,7 @@ build for linux arm64: GOARCH: "arm64" script: - go get github.com/rakyll/statik - - tar -xzvf lattice.tar.gz + - tar -xvf lattice.tar.gz - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_linux_arm64" artifacts: @@ -140,7 +140,7 @@ build for darwin amd64: GOARCH: "amd64" script: - go get github.com/rakyll/statik - - tar -xzvf lattice.tar.gz + - tar -xvf lattice.tar.gz - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" artifacts: @@ -156,7 +156,7 @@ build for darwin arm64: GOARCH: "arm64" script: - go get github.com/rakyll/statik - - tar -xzvf lattice.tar.gz + - tar -xvf lattice.tar.gz - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_arm64" artifacts: From 61d558de3cbaedff357fd7081076d628411e2924 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 17:40:28 -0600 Subject: [PATCH 108/222] update tests, actually install statik smh my head --- .gitlab/.gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 462977fa6..0f1f4bc3c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -68,7 +68,7 @@ run jest tests: run go tests: stage: test - image: golang:1.16.9 + image: golang:1.16.10 extends: .go-cache script: - echo "Running featurebase unit tests..." @@ -79,7 +79,7 @@ run go tests: run go tests with output: stage: test - image: golang:1.16.9 + image: golang:1.16.10 script: - echo "Running featurebase unit tests to capture JSON output..." - go test -json > test-report.out @@ -108,6 +108,7 @@ build for linux amd64: GOARCH: "amd64" script: - go get github.com/rakyll/statik + - go install github.com/rakyll/statik@latest - tar -xvf lattice.tar.gz - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_linux_amd64" @@ -124,6 +125,7 @@ build for linux arm64: GOARCH: "arm64" script: - go get github.com/rakyll/statik + - go install github.com/rakyll/statik@latest - tar -xvf lattice.tar.gz - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_linux_arm64" @@ -140,6 +142,7 @@ build for darwin amd64: GOARCH: "amd64" script: - go get github.com/rakyll/statik + - go install github.com/rakyll/statik@latest - tar -xvf lattice.tar.gz - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" @@ -156,6 +159,7 @@ build for darwin arm64: GOARCH: "arm64" script: - go get github.com/rakyll/statik + - go install github.com/rakyll/statik@latest - tar -xvf lattice.tar.gz - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_arm64" From 640d12cb6cb03b80aed96a531acdbd338cf0c907 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 17:54:26 -0600 Subject: [PATCH 109/222] debug by print statmeentS --- .gitlab/.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 0f1f4bc3c..9f09e035e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -143,8 +143,10 @@ build for darwin amd64: script: - go get github.com/rakyll/statik - go install github.com/rakyll/statik@latest + - /go/bin/statik -h - tar -xvf lattice.tar.gz - /go/bin/statik -src=lattice + - ls - make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: From 1337a9dfaa4d12775d4ebcc7e4bcced4985d6248 Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 18:12:27 -0600 Subject: [PATCH 110/222] more debugging by print statement --- .gitlab/.gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 9f09e035e..23ce3b36b 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -141,12 +141,14 @@ build for darwin amd64: GOOS: "darwin" GOARCH: "amd64" script: + - tar -xvf lattice.tar.gz + - ls - go get github.com/rakyll/statik - go install github.com/rakyll/statik@latest - - /go/bin/statik -h - - tar -xvf lattice.tar.gz + - ls -al /go + - which statik + - ls -al /go/bin - /go/bin/statik -src=lattice - - ls - make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: From b28c9feef04f3881b214fb62636f6156ffd6e9df Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 11 Nov 2021 18:28:20 -0600 Subject: [PATCH 111/222] more print debug --- .gitlab/.gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 23ce3b36b..b7040a3c1 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -141,12 +141,10 @@ build for darwin amd64: GOOS: "darwin" GOARCH: "amd64" script: + - rm -r lattice - tar -xvf lattice.tar.gz - - ls - go get github.com/rakyll/statik - go install github.com/rakyll/statik@latest - - ls -al /go - - which statik - ls -al /go/bin - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" From a214ac5dedb97cbe8b00b94af017a2f164ac8a5e Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 11:46:52 -0600 Subject: [PATCH 112/222] verbose statik install --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index b7040a3c1..538d32e97 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -144,7 +144,7 @@ build for darwin amd64: - rm -r lattice - tar -xvf lattice.tar.gz - go get github.com/rakyll/statik - - go install github.com/rakyll/statik@latest + - go install -v github.com/rakyll/statik@latest - ls -al /go/bin - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" From 14e9fb1ff14897ef7c9f1af57c0af12fa9275ad1 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 12:02:45 -0600 Subject: [PATCH 113/222] statik h --- .gitlab/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 538d32e97..d9e79826d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -144,9 +144,9 @@ build for darwin amd64: - rm -r lattice - tar -xvf lattice.tar.gz - go get github.com/rakyll/statik - - go install -v github.com/rakyll/statik@latest - - ls -al /go/bin - - /go/bin/statik -src=lattice + - go install github.com/rakyll/statik@latest + - statik -h + - statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: From 67835f746d3988d192f6a2883c6ca863ebe5b3f5 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 12:17:09 -0600 Subject: [PATCH 114/222] please work --- .gitlab/.gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d9e79826d..3263764ac 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -143,10 +143,9 @@ build for darwin amd64: script: - rm -r lattice - tar -xvf lattice.tar.gz - - go get github.com/rakyll/statik - - go install github.com/rakyll/statik@latest - - statik -h - - statik -src=lattice + - sudo go get github.com/rakyll/statik + - /go/bin/statik -h + - /go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: From 49ce895c4d963c4be8a4a1aebfe6f0f601aeea40 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 12:19:27 -0600 Subject: [PATCH 115/222] change path --- .gitlab/.gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3263764ac..3a135677e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -143,9 +143,8 @@ build for darwin amd64: script: - rm -r lattice - tar -xvf lattice.tar.gz - - sudo go get github.com/rakyll/statik - - /go/bin/statik -h - - /go/bin/statik -src=lattice + - go get github.com/rakyll/statik + - ~/go/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: From 948928c06efaab303e8501e8c3ee466b67b49ae0 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 12:37:18 -0600 Subject: [PATCH 116/222] set gopath explicitly --- .gitlab/.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3a135677e..767ecfda1 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -140,7 +140,9 @@ build for darwin amd64: variables: GOOS: "darwin" GOARCH: "amd64" + GOPATH: "~/go/" script: + - mkdir -p ~/go/bin - rm -r lattice - tar -xvf lattice.tar.gz - go get github.com/rakyll/statik From 220515bbd4ad605f00d30b54129dabf81da9fb8c Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 12:53:33 -0600 Subject: [PATCH 117/222] "update path" --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 767ecfda1..bee4e8cda 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -140,13 +140,13 @@ build for darwin amd64: variables: GOOS: "darwin" GOARCH: "amd64" - GOPATH: "~/go/" + GOPATH: "/go/" script: - mkdir -p ~/go/bin - rm -r lattice - tar -xvf lattice.tar.gz - go get github.com/rakyll/statik - - ~/go/bin/statik -src=lattice + - $GOPATH/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: From 91271795acf4619e582df4f8b6b0a065458a71a7 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 13:07:42 -0600 Subject: [PATCH 118/222] f --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bee4e8cda..a4499efd5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -140,7 +140,7 @@ build for darwin amd64: variables: GOOS: "darwin" GOARCH: "amd64" - GOPATH: "/go/" + GOPATH: "/go" script: - mkdir -p ~/go/bin - rm -r lattice From 0d8ecfbcb5b13f38e3a7e5a821147f2628def368 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 13:22:12 -0600 Subject: [PATCH 119/222] be more explicit? --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index a4499efd5..2a8154489 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -142,10 +142,10 @@ build for darwin amd64: GOARCH: "amd64" GOPATH: "/go" script: - - mkdir -p ~/go/bin + - mkdir -p $GOPATH - rm -r lattice - tar -xvf lattice.tar.gz - - go get github.com/rakyll/statik + - go get -v -u github.com/rakyll/statik - $GOPATH/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" artifacts: From 9ff56e032ec764b6ae1fbffbdfbd70cf825e25ea Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 13:37:57 -0600 Subject: [PATCH 120/222] pls --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 2a8154489..bc91bd649 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -146,6 +146,7 @@ build for darwin amd64: - rm -r lattice - tar -xvf lattice.tar.gz - go get -v -u github.com/rakyll/statik + - go install github.com/rakyll/statik@latest - $GOPATH/bin/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" artifacts: From de0e8cc0f9d9e88967fa57e3d63407ae20b93ba6 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 13:52:39 -0600 Subject: [PATCH 121/222] maybe go bin is in a different spot? --- .gitlab/.gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index bc91bd649..20fae519e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -140,14 +140,14 @@ build for darwin amd64: variables: GOOS: "darwin" GOARCH: "amd64" - GOPATH: "/go" + GOBIN: "/go/bin" script: - - mkdir -p $GOPATH + - mkdir -p $GOBIN - rm -r lattice - tar -xvf lattice.tar.gz - go get -v -u github.com/rakyll/statik - - go install github.com/rakyll/statik@latest - - $GOPATH/bin/statik -src=lattice + - go install -v github.com/rakyll/statik@latest + - $GOBIN/statik -src=lattice - make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: From 4995888f00e245bf4094c5e0aea7e301f9cbe206 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 14:11:56 -0600 Subject: [PATCH 122/222] the problem was i was installing it for a different architecture smh --- .gitlab/.gitlab-ci.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 20fae519e..83bb9be05 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -137,18 +137,12 @@ build for darwin amd64: stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 extends: .go-cache - variables: - GOOS: "darwin" - GOARCH: "amd64" - GOBIN: "/go/bin" script: - - mkdir -p $GOBIN - rm -r lattice - tar -xvf lattice.tar.gz - go get -v -u github.com/rakyll/statik - - go install -v github.com/rakyll/statik@latest - $GOBIN/statik -src=lattice - - make build FLAGS="-o featurebase_darwin_amd64" + - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: - featurebase_darwin_amd64 From fd1a2d2e782adb9aef4d09f922d20a66f32e9516 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 14:26:27 -0600 Subject: [PATCH 123/222] smh --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 83bb9be05..d91592dd9 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -141,7 +141,7 @@ build for darwin amd64: - rm -r lattice - tar -xvf lattice.tar.gz - go get -v -u github.com/rakyll/statik - - $GOBIN/statik -src=lattice + - /go/bin/statik -src=lattice - GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64" artifacts: paths: From 541df0fe3733b7e571c952b26b7a464e46f45477 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 14:40:05 -0600 Subject: [PATCH 124/222] is it because of the go cache? --- .gitlab/.gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index d91592dd9..29cb2b05e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -136,7 +136,6 @@ build for linux arm64: build for darwin amd64: stage: build image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - extends: .go-cache script: - rm -r lattice - tar -xvf lattice.tar.gz From d1345ca265cfb788479b83183f8c36aa75dd73a7 Mon Sep 17 00:00:00 2001 From: reesporte Date: Sun, 14 Nov 2021 15:00:28 -0600 Subject: [PATCH 125/222] build all with golang:1.16.10 --- .gitlab/.gitlab-ci.yml | 50 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 29cb2b05e..5d07a1dc9 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -77,6 +77,18 @@ run go tests: paths: - coverage.out +run go tests future: + stage: test + image: golang:1.17.3 + extends: .go-cache + script: + - echo "Running featurebase unit tests..." + - ./cover-everything.sh + artifacts: + paths: + - coverage.out + + run go tests with output: stage: test image: golang:1.16.10 @@ -102,40 +114,32 @@ upload to sonarcloud: build for linux amd64: stage: build image: golang:1.16.10 - extends: .go-cache - variables: - GOOS: "linux" - GOARCH: "amd64" script: - - go get github.com/rakyll/statik - - go install github.com/rakyll/statik@latest + - rm -r lattice - tar -xvf lattice.tar.gz + - go get -v -u github.com/rakyll/statik - /go/bin/statik -src=lattice - - make build FLAGS="-o featurebase_linux_amd64" + - GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64" artifacts: paths: - featurebase_linux_amd64 build for linux arm64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - extends: .go-cache - variables: - GOOS: "linux" - GOARCH: "arm64" + image: golang:1.16.10 script: - - go get github.com/rakyll/statik - - go install github.com/rakyll/statik@latest + - rm -r lattice - tar -xvf lattice.tar.gz + - go get -v -u github.com/rakyll/statik - /go/bin/statik -src=lattice - - make build FLAGS="-o featurebase_linux_arm64" + - GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64" artifacts: paths: - featurebase_linux_arm64 build for darwin amd64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + image: golang:1.16.10 script: - rm -r lattice - tar -xvf lattice.tar.gz @@ -148,24 +152,20 @@ build for darwin amd64: build for darwin arm64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 - extends: .go-cache - variables: - GOOS: "darwin" - GOARCH: "arm64" + image: golang:1.16.10 script: - - go get github.com/rakyll/statik - - go install github.com/rakyll/statik@latest + - rm -r lattice - tar -xvf lattice.tar.gz + - go get -v -u github.com/rakyll/statik - /go/bin/statik -src=lattice - - make build FLAGS="-o featurebase_darwin_arm64" + - GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64" artifacts: paths: - featurebase_darwin_arm64 package for linux amd64: stage: build - image: registry.gitlab.com/molecula/featurebase/builder:0.0.3 + image: golang:1.16.10 variables: GOOS: "linux" GOARCH: "amd64" From 0a4e9fe8cf8273d17244b28b273ddcad4c18aafb Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 10:16:43 -0600 Subject: [PATCH 126/222] move lattice build dir to right location --- .gitlab/.gitlab-ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 5d07a1dc9..cb2d6546c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -42,12 +42,17 @@ build lattice: variables: CI: "false" script: - - echo "Building lattice..." + - # install and build lattice - cd lattice - yarn install - yarn build + - # move the build dir out and rename + - mv build ../ - cd ../ - - tar -czvf lattice.tar.gz lattice/build + - rm -r lattice + - mv build lattice + - # zip it up + - tar -czvf lattice.tar.gz lattice artifacts: paths: - lattice.tar.gz From ed5525c83675c19905eb5f9be8ebfe3080a4e7dc Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 10:19:51 -0600 Subject: [PATCH 127/222] remove long lines i guess --- .gitlab/.gitlab-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index cb2d6546c..ccd710889 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -42,16 +42,13 @@ build lattice: variables: CI: "false" script: - - # install and build lattice - cd lattice - yarn install - yarn build - - # move the build dir out and rename - mv build ../ - cd ../ - rm -r lattice - mv build lattice - - # zip it up - tar -czvf lattice.tar.gz lattice artifacts: paths: From 7de69df4241e7c0f25369c8ede56a191df0195ee Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 15 Nov 2021 10:54:13 -0600 Subject: [PATCH 128/222] added protection against trailing slash --- cmd/roaring-migrate/main.go | 3 +++ cover-everything.sh | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 627d56ef2..e12348e56 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -243,6 +243,9 @@ func copyFile(src, dest string) error { } func Migrate(dataDir, backupPath string) error { + if strings.HasSuffix(dataDir, "/") { + dataDir = dataDir[:len(dataDir)-1] + } err := os.MkdirAll(backupPath, 0777) if err != nil { return err diff --git a/cover-everything.sh b/cover-everything.sh index f34640cfb..500c9013b 100755 --- a/cover-everything.sh +++ b/cover-everything.sh @@ -2,7 +2,8 @@ # actually get test coverage for every single package and subpackage # very slow but oh well what are you gonna do, not test things? +# note it skips the roaring migrate echo "mode: atomic" > coverage.out -for pkg in $(go list all | grep featurebase); do +for pkg in $(go list all | grep featurebase | grep -v roaring-migrate); do go test -coverprofile=pkgcoverage.out -covermode=atomic $pkg; tail -n +2 pkgcoverage.out >> coverage.out; done From f0a6ebae8f58636ff889f6e0e15ff42cd2be8cf0 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 15 Nov 2021 14:51:57 -0600 Subject: [PATCH 129/222] ingest and delete for samsung workflow --- .gitlab/ingestWorkload.sh | 88 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 .gitlab/ingestWorkload.sh diff --git a/.gitlab/ingestWorkload.sh b/.gitlab/ingestWorkload.sh new file mode 100755 index 000000000..2502dba2a --- /dev/null +++ b/.gitlab/ingestWorkload.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# To run: +# ./ingestWorkload.sh {Path for featurebase binary} {Path for directory with csv files} {initialize flag} + +function delete_field { + if (($INITIALIZE == 0)); + then + curl -XDELETE $HOST/index/$INDEX/field/$FIELD + fi +} + +# Script to replicate samsung workload of deleting and re-ingesting fields every night +# outline delete and re-ingest workload +function ingest_int_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_time_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_set_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +# path for featurebase binary +FEATUREBASE_PATH=$1 +shift + +# path for directory with csv directory files for all fields to be ingested +CSV_DIR_PATH=$1 +shift + +# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing +INITIALIZE=$1 +shift + +# get a list of csv files in the directory +CSV_FILES=`ls $CSV_DIR_PATH/*.csv` + +# featurebase host +HOST="localhost:10101" +# assign index name +INDEX="samsung" +if (($INITIALIZE == 1)); +then + curl -XPOST $HOST/index/$INDEX +fi + +# perform delete and re-ingest for all fields +for CSV_FILE in ${CSV_FILES[@]} + do + # get field name from csv file path + FIELD="$(basename $CSV_FILE .csv)" + if [[ "$FIELD" == *"age"* ]]; + then + MAX=100 + echo $CSV_FILE $MAX + ingest_int_field + elif [[ "$FIELD" == *"identifier"* ]]; + then + MAX=$((2**63 - 1)) # compute max value for 64bit + echo $CSV_FILE $MAX + ingest_int_field + elif [[ "$FIELD" == *"ip"* ]]; + then + MAX=$((2**31 - 1)) # compute max value for 32bit + echo $CSV_FILE $MAX + ingest_int_field + elif [[ "$FIELD" == *"time"* ]]; + then + echo $CSV_FILE + ingest_time_field + else + echo $CSV_FILE + ingest_set_field + fi + done + + + From 96dfe845d334cd415e7eb4c04104727a372d385b Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 15:04:27 -0600 Subject: [PATCH 130/222] fix presentation of timestamps from a distinct pql call --- executor.go | 19 +++++++++++++++++++ server/pg.go | 21 ++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index f99a0f667..6523ce925 100644 --- a/executor.go +++ b/executor.go @@ -322,6 +322,8 @@ func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { safe[i] = v.Clone() } out.Results = append(out.Results, safe) + case DistinctTimestamp: + out.Results = append(out.Results, x) default: panic(fmt.Sprintf("handle %T here", v)) } @@ -1498,6 +1500,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if field == nil { return nil, ErrFieldNotFound } + bsig := field.bsiGroup(fieldName) if bsig == nil { result = &Row{ @@ -1534,9 +1537,25 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if bsig == nil { return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap) } + if field.Options().Type == FieldTypeTimestamp { + r, err := executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) + if err != nil { + return nil, err + } + results := make([]string, len(r.Pos.Columns())) + for i, val := range r.Pos.Columns() { + results[i] = time.Unix(0, (int64(val)+int64(bsig.Base))*TimeUnitNanos(field.options.TimeUnit)).UTC().Format(time.RFC3339Nano) + } + return DistinctTimestamp{Name: fieldName, Values: results}, nil + } return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) } +type DistinctTimestamp struct { + Values []string + Name string +} + func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) { index := idx.Name() tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) diff --git a/server/pg.go b/server/pg.go index c2522f286..2a55bc3e1 100644 --- a/server/pg.go +++ b/server/pg.go @@ -155,6 +155,24 @@ type PilosaQueryHandler struct { sqlVersion SqlVersion } +func pgWriteDistinctTimestamp(w pg.QueryResultWriter, val pilosa.DistinctTimestamp) error { + err := w.WriteHeader(pg.ColumnInfo{ + Name: val.Name, + Type: pg.TypeCharoid, + }) + if err != nil { + return errors.Wrap(err, "writing result header") + } + + for _, k := range val.Values { + err = w.WriteRowText(k) + if err != nil { + return errors.Wrap(err, "writing key") + } + } + return nil +} + func pgWriteRow(w pg.QueryResultWriter, row *pilosa.Row) error { err := w.WriteHeader(pg.ColumnInfo{ Name: "_id", @@ -551,7 +569,8 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { } return nil - + case pilosa.DistinctTimestamp: + return pgWriteDistinctTimestamp(w, result) case nil: return nil From 9e570c9e84424b8af694f65d929e92f09f5c2dfb Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 15:59:51 -0600 Subject: [PATCH 131/222] fix presentation of timestamps from a groupby pql call --- executor.go | 17 ++++++++++++----- server/pg.go | 6 +++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/executor.go b/executor.go index 6523ce925..686379609 100644 --- a/executor.go +++ b/executor.go @@ -3064,10 +3064,11 @@ func applyLimitAndOffsetToGroupByResult(c *pql.Call, results []GroupCount) ([]Gr // FieldRow is used to distinguish rows in a group by result. type FieldRow struct { - Field string `json:"field"` - RowID uint64 `json:"rowID"` - RowKey string `json:"rowKey,omitempty"` - Value *int64 `json:"value,omitempty"` + Field string `json:"field"` + RowID uint64 `json:"rowID"` + RowKey string `json:"rowKey,omitempty"` + Value *int64 `json:"value,omitempty"` + FieldOptions *FieldOptions `json:"-"` } func (fr *FieldRow) Clone() (clone *FieldRow) { @@ -3081,6 +3082,11 @@ func (fr *FieldRow) Clone() (clone *FieldRow) { v := *fr.Value clone.Value = &v } + if fr.FieldOptions != nil { + // deep copy, for Extra Safety + v := *fr.FieldOptions + clone.FieldOptions = &v + } return } @@ -7738,6 +7744,8 @@ func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children return nil, newNotFoundError(ErrFieldNotFound, fieldName) } gbi.fields[i].Field = fieldName + options := field.Options() + gbi.fields[i].FieldOptions = &options switch field.Type() { case FieldTypeSet, FieldTypeMutex, FieldTypeBool: @@ -7971,7 +7979,6 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool ret.Group = make([]FieldRow, len(gbi.rows)) copy(ret.Group, gbi.fields) for i, r := range gbi.rows { - ret.Group[i].RowID = r.id ret.Group[i].Value = r.value } diff --git a/server/pg.go b/server/pg.go index 2a55bc3e1..da6dfd633 100644 --- a/server/pg.go +++ b/server/pg.go @@ -331,7 +331,11 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error var v string switch { case g.Value != nil: - v = strconv.FormatInt(*g.Value, 10) + if g.FieldOptions.Type == pilosa.FieldTypeTimestamp { + v = time.Unix(0, (int64(*g.Value)+int64(g.FieldOptions.Base))*pilosa.TimeUnitNanos(g.FieldOptions.TimeUnit)).UTC().Format(time.RFC3339Nano) + } else { + v = strconv.FormatInt(*g.Value, 10) + } case g.RowKey != "": v = g.RowKey default: From 269526348e7326eae24e81b486ffd42ec38c98cd Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 17:03:40 -0600 Subject: [PATCH 132/222] update test to ignore FieldOptions field --- test/pilosa.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index 925213cd1..0703e394c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/http" @@ -315,7 +315,27 @@ func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } for i, result := range results { - if !reflect.DeepEqual(expected[i], result) { + // have to check each field Row individually because FieldOptions is getting set + for j := range expected[i].Group { + // Field:"ppa", RowID:0x3, RowKey:"", Value:(*int64)(nil), FieldOptions: + if !reflect.DeepEqual(expected[i].Group[j].Field, result.Group[j].Field) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].RowKey, result.Group[j].RowKey) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].Value, result.Group[j].Value) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + } + + if !reflect.DeepEqual(expected[i].Count, result.Count) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Agg, result.Agg) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].DecimalAgg, result.DecimalAgg) { t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) } } From 8770ce245effd6dcf61ee441d38954efc2131f12 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 15 Nov 2021 14:51:57 -0600 Subject: [PATCH 133/222] ingest and delete for samsung workflow --- .gitlab/ingestWorkload.sh | 88 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 .gitlab/ingestWorkload.sh diff --git a/.gitlab/ingestWorkload.sh b/.gitlab/ingestWorkload.sh new file mode 100755 index 000000000..2502dba2a --- /dev/null +++ b/.gitlab/ingestWorkload.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# To run: +# ./ingestWorkload.sh {Path for featurebase binary} {Path for directory with csv files} {initialize flag} + +function delete_field { + if (($INITIALIZE == 0)); + then + curl -XDELETE $HOST/index/$INDEX/field/$FIELD + fi +} + +# Script to replicate samsung workload of deleting and re-ingesting fields every night +# outline delete and re-ingest workload +function ingest_int_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_time_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +function ingest_set_field { + delete_field + curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}' + $FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE +} + +# path for featurebase binary +FEATUREBASE_PATH=$1 +shift + +# path for directory with csv directory files for all fields to be ingested +CSV_DIR_PATH=$1 +shift + +# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing +INITIALIZE=$1 +shift + +# get a list of csv files in the directory +CSV_FILES=`ls $CSV_DIR_PATH/*.csv` + +# featurebase host +HOST="localhost:10101" +# assign index name +INDEX="samsung" +if (($INITIALIZE == 1)); +then + curl -XPOST $HOST/index/$INDEX +fi + +# perform delete and re-ingest for all fields +for CSV_FILE in ${CSV_FILES[@]} + do + # get field name from csv file path + FIELD="$(basename $CSV_FILE .csv)" + if [[ "$FIELD" == *"age"* ]]; + then + MAX=100 + echo $CSV_FILE $MAX + ingest_int_field + elif [[ "$FIELD" == *"identifier"* ]]; + then + MAX=$((2**63 - 1)) # compute max value for 64bit + echo $CSV_FILE $MAX + ingest_int_field + elif [[ "$FIELD" == *"ip"* ]]; + then + MAX=$((2**31 - 1)) # compute max value for 32bit + echo $CSV_FILE $MAX + ingest_int_field + elif [[ "$FIELD" == *"time"* ]]; + then + echo $CSV_FILE + ingest_time_field + else + echo $CSV_FILE + ingest_set_field + fi + done + + + From 6413ed3228b5e19f342f94adffaba7b8a86f62fe Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 15 Nov 2021 10:54:13 -0600 Subject: [PATCH 134/222] added protection against trailing slash --- cmd/roaring-migrate/main.go | 3 +++ cover-everything.sh | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 627d56ef2..e12348e56 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -243,6 +243,9 @@ func copyFile(src, dest string) error { } func Migrate(dataDir, backupPath string) error { + if strings.HasSuffix(dataDir, "/") { + dataDir = dataDir[:len(dataDir)-1] + } err := os.MkdirAll(backupPath, 0777) if err != nil { return err diff --git a/cover-everything.sh b/cover-everything.sh index f34640cfb..500c9013b 100755 --- a/cover-everything.sh +++ b/cover-everything.sh @@ -2,7 +2,8 @@ # actually get test coverage for every single package and subpackage # very slow but oh well what are you gonna do, not test things? +# note it skips the roaring migrate echo "mode: atomic" > coverage.out -for pkg in $(go list all | grep featurebase); do +for pkg in $(go list all | grep featurebase | grep -v roaring-migrate); do go test -coverprofile=pkgcoverage.out -covermode=atomic $pkg; tail -n +2 pkgcoverage.out >> coverage.out; done From f5afb7a3eda222ac50f165cf2c95b370b8116fc4 Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 17:03:40 -0600 Subject: [PATCH 135/222] update test to ignore FieldOptions field --- test/pilosa.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index 925213cd1..0703e394c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/http" @@ -315,7 +315,27 @@ func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } for i, result := range results { - if !reflect.DeepEqual(expected[i], result) { + // have to check each field Row individually because FieldOptions is getting set + for j := range expected[i].Group { + // Field:"ppa", RowID:0x3, RowKey:"", Value:(*int64)(nil), FieldOptions: + if !reflect.DeepEqual(expected[i].Group[j].Field, result.Group[j].Field) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].RowKey, result.Group[j].RowKey) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].Value, result.Group[j].Value) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + } + + if !reflect.DeepEqual(expected[i].Count, result.Count) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Agg, result.Agg) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].DecimalAgg, result.DecimalAgg) { t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) } } From 6d6cf7e51f53abaabcee8c6b396e54f8d5eed3ec Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 15:04:27 -0600 Subject: [PATCH 136/222] fix presentation of timestamps from a distinct pql call --- executor.go | 19 +++++++++++++++++++ server/pg.go | 21 ++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index f99a0f667..6523ce925 100644 --- a/executor.go +++ b/executor.go @@ -322,6 +322,8 @@ func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { safe[i] = v.Clone() } out.Results = append(out.Results, safe) + case DistinctTimestamp: + out.Results = append(out.Results, x) default: panic(fmt.Sprintf("handle %T here", v)) } @@ -1498,6 +1500,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if field == nil { return nil, ErrFieldNotFound } + bsig := field.bsiGroup(fieldName) if bsig == nil { result = &Row{ @@ -1534,9 +1537,25 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if bsig == nil { return executeDistinctShardSet(ctx, qcx, idx, fieldName, shard, filterBitmap) } + if field.Options().Type == FieldTypeTimestamp { + r, err := executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) + if err != nil { + return nil, err + } + results := make([]string, len(r.Pos.Columns())) + for i, val := range r.Pos.Columns() { + results[i] = time.Unix(0, (int64(val)+int64(bsig.Base))*TimeUnitNanos(field.options.TimeUnit)).UTC().Format(time.RFC3339Nano) + } + return DistinctTimestamp{Name: fieldName, Values: results}, nil + } return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) } +type DistinctTimestamp struct { + Values []string + Name string +} + func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldName string, shard uint64, filterBitmap *roaring.Bitmap) (result *Row, err0 error) { index := idx.Name() tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) diff --git a/server/pg.go b/server/pg.go index c2522f286..2a55bc3e1 100644 --- a/server/pg.go +++ b/server/pg.go @@ -155,6 +155,24 @@ type PilosaQueryHandler struct { sqlVersion SqlVersion } +func pgWriteDistinctTimestamp(w pg.QueryResultWriter, val pilosa.DistinctTimestamp) error { + err := w.WriteHeader(pg.ColumnInfo{ + Name: val.Name, + Type: pg.TypeCharoid, + }) + if err != nil { + return errors.Wrap(err, "writing result header") + } + + for _, k := range val.Values { + err = w.WriteRowText(k) + if err != nil { + return errors.Wrap(err, "writing key") + } + } + return nil +} + func pgWriteRow(w pg.QueryResultWriter, row *pilosa.Row) error { err := w.WriteHeader(pg.ColumnInfo{ Name: "_id", @@ -551,7 +569,8 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { } return nil - + case pilosa.DistinctTimestamp: + return pgWriteDistinctTimestamp(w, result) case nil: return nil From b3b536505dad34a943982571a288ddc939627e3c Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 15:59:51 -0600 Subject: [PATCH 137/222] fix presentation of timestamps from a groupby pql call --- executor.go | 17 ++++++++++++----- server/pg.go | 6 +++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/executor.go b/executor.go index 6523ce925..686379609 100644 --- a/executor.go +++ b/executor.go @@ -3064,10 +3064,11 @@ func applyLimitAndOffsetToGroupByResult(c *pql.Call, results []GroupCount) ([]Gr // FieldRow is used to distinguish rows in a group by result. type FieldRow struct { - Field string `json:"field"` - RowID uint64 `json:"rowID"` - RowKey string `json:"rowKey,omitempty"` - Value *int64 `json:"value,omitempty"` + Field string `json:"field"` + RowID uint64 `json:"rowID"` + RowKey string `json:"rowKey,omitempty"` + Value *int64 `json:"value,omitempty"` + FieldOptions *FieldOptions `json:"-"` } func (fr *FieldRow) Clone() (clone *FieldRow) { @@ -3081,6 +3082,11 @@ func (fr *FieldRow) Clone() (clone *FieldRow) { v := *fr.Value clone.Value = &v } + if fr.FieldOptions != nil { + // deep copy, for Extra Safety + v := *fr.FieldOptions + clone.FieldOptions = &v + } return } @@ -7738,6 +7744,8 @@ func newGroupByIterator(executor *executor, qcx *Qcx, rowIDs []RowIDs, children return nil, newNotFoundError(ErrFieldNotFound, fieldName) } gbi.fields[i].Field = fieldName + options := field.Options() + gbi.fields[i].FieldOptions = &options switch field.Type() { case FieldTypeSet, FieldTypeMutex, FieldTypeBool: @@ -7971,7 +7979,6 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool ret.Group = make([]FieldRow, len(gbi.rows)) copy(ret.Group, gbi.fields) for i, r := range gbi.rows { - ret.Group[i].RowID = r.id ret.Group[i].Value = r.value } diff --git a/server/pg.go b/server/pg.go index 2a55bc3e1..da6dfd633 100644 --- a/server/pg.go +++ b/server/pg.go @@ -331,7 +331,11 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error var v string switch { case g.Value != nil: - v = strconv.FormatInt(*g.Value, 10) + if g.FieldOptions.Type == pilosa.FieldTypeTimestamp { + v = time.Unix(0, (int64(*g.Value)+int64(g.FieldOptions.Base))*pilosa.TimeUnitNanos(g.FieldOptions.TimeUnit)).UTC().Format(time.RFC3339Nano) + } else { + v = strconv.FormatInt(*g.Value, 10) + } case g.RowKey != "": v = g.RowKey default: From 9bbd946ac5b1ec56f3540940d92d07c4634c7433 Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 15 Nov 2021 17:03:40 -0600 Subject: [PATCH 138/222] update test to ignore FieldOptions field --- test/pilosa.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/pilosa.go b/test/pilosa.go index 925213cd1..0703e394c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/encoding/proto" "github.com/molecula/featurebase/v2/http" @@ -315,7 +315,27 @@ func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } for i, result := range results { - if !reflect.DeepEqual(expected[i], result) { + // have to check each field Row individually because FieldOptions is getting set + for j := range expected[i].Group { + // Field:"ppa", RowID:0x3, RowKey:"", Value:(*int64)(nil), FieldOptions: + if !reflect.DeepEqual(expected[i].Group[j].Field, result.Group[j].Field) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].RowKey, result.Group[j].RowKey) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Group[j].Value, result.Group[j].Value) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + } + + if !reflect.DeepEqual(expected[i].Count, result.Count) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].Agg, result.Agg) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + if !reflect.DeepEqual(expected[i].DecimalAgg, result.DecimalAgg) { t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) } } From da783d97efc1ee289d773d2d2f31a38398bd3f5f Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 16 Nov 2021 09:14:15 -0600 Subject: [PATCH 139/222] add test for pgWriteDistinctTimestamp --- server/pg_internal_test.go | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 server/pg_internal_test.go diff --git a/server/pg_internal_test.go b/server/pg_internal_test.go new file mode 100644 index 000000000..3b83d032e --- /dev/null +++ b/server/pg_internal_test.go @@ -0,0 +1,53 @@ +package server + +import ( + "testing" + + pilosa "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/pg" +) + +// pg_internal_test.go tests unexported methods from server/pg.go + +// TestQueryResultWriter implements the QueryResultWriter interface for testing +type TestQueryResultWriter struct { + Header []pg.ColumnInfo + RowText []string + TagTag string +} + +func (t *TestQueryResultWriter) WriteHeader(headers ...pg.ColumnInfo) error { + for _, header := range headers { + t.Header = append(t.Header, header) + } + return nil +} + +func (t *TestQueryResultWriter) WriteRowText(rowTexts ...string) error { + for _, rowText := range rowTexts { + t.RowText = append(t.RowText, rowText) + } + return nil +} + +func (t *TestQueryResultWriter) Tag(tag string) { + t.TagTag = tag +} + +func TestPgWriteDistinctTimestamp(t *testing.T) { + w := TestQueryResultWriter{} + expected := pilosa.DistinctTimestamp{Name: "test", Values: []string{"date1", "date2", "date3"}} + pgWriteDistinctTimestamp(&w, expected) + if w.Header[0].Name != expected.Name { + t.Fatalf("Header Name is wrong. got %v, want %v", w.Header[0], expected.Name) + } + if w.Header[0].Type != pg.TypeCharoid { + t.Fatalf("Header Type is wrong. got %v, want %v", w.Header[0].Type, pg.TypeCharoid) + } + for i, value := range w.RowText { + if value != expected.Values[i] { + t.Fatalf("Value not written properly. got %v, want %v", value, expected.Values[i]) + } + } + +} From 9e041803fb16f6e240e154ac16b17750eeddf4ca Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 16 Nov 2021 09:20:44 -0600 Subject: [PATCH 140/222] add license --- server/pg_internal_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/pg_internal_test.go b/server/pg_internal_test.go index 3b83d032e..75024d509 100644 --- a/server/pg_internal_test.go +++ b/server/pg_internal_test.go @@ -1,3 +1,17 @@ +// Copyright 2021 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package server import ( From 4c966dee22695127ce4ff48423d41973789b8db8 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 16 Nov 2021 12:08:46 -0600 Subject: [PATCH 141/222] removed filter --- cover-everything.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cover-everything.sh b/cover-everything.sh index 500c9013b..808fc9997 100755 --- a/cover-everything.sh +++ b/cover-everything.sh @@ -4,6 +4,6 @@ # very slow but oh well what are you gonna do, not test things? # note it skips the roaring migrate echo "mode: atomic" > coverage.out -for pkg in $(go list all | grep featurebase | grep -v roaring-migrate); do +for pkg in $(go list all | grep featurebase ); do go test -coverprofile=pkgcoverage.out -covermode=atomic $pkg; tail -n +2 pkgcoverage.out >> coverage.out; done From dcab1a6708b30bad59d55221655e29b96eaeec63 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 16 Nov 2021 14:23:35 -0600 Subject: [PATCH 142/222] unbreak featurebase holder subcommand The "just open the holder" subcommand doesn't work the way it used to, because now that we rely on etcd to open a holder, trying to open a holder without things set up just coredumps. Step 1: Fix that. Step 2: Also add a test that covers it so we don't get bitrotted again. Step 3: Remove an unrelated stale comment that doesn't deserve its own commit log, having to do with an option that no longer exists which is no longer being set right under the comment saying we set it. --- server.go | 18 +++++------------- server/server.go | 3 ++- server/server_test.go | 8 ++++++++ test/pilosa.go | 2 -- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/server.go b/server.go index 8058c25f4..bcb86f119 100644 --- a/server.go +++ b/server.go @@ -538,25 +538,17 @@ func (s *Server) SetAPI(api *API) { // UpAndDown brings the server up minimally and shuts it down // again; basically, it exists for testing holder open and close. func (s *Server) UpAndDown() error { - s.logger.Infof("open server. PID %v", os.Getpid()) - // Log startup err := s.holder.logStartup() if err != nil { log.Println(errors.Wrap(err, "logging startup")) } - - // Open holder. - if err := s.holder.Open(); err != nil { - return errors.Wrap(err, "opening Holder") + s.logger.Infof("open server. PID %v", os.Getpid()) + if err = s.Open(); err != nil { + return errors.Wrap(err, "starting server") } - - errh := s.holder.Close() - if errh != nil { - return errors.Wrap(errh, "closing holder") - } - - return nil + err = s.Close() + return errors.Wrap(err, "shutting down server") } // Open opens and initializes the server. diff --git a/server/server.go b/server/server.go index 3feaf4582..29eb20f1d 100644 --- a/server/server.go +++ b/server/server.go @@ -293,6 +293,7 @@ func (m *Command) UpAndDown() (err error) { if err != nil { return errors.Wrap(err, "setting up server") } + m.logger.Infof("bringing server up and shutting it down immediately") go func() { err := m.Handler.Serve() @@ -306,7 +307,7 @@ func (m *Command) UpAndDown() (err error) { return errors.Wrap(err, "bringing server up and down") } - m.logger.Errorf("brought up and shut down again") + m.logger.Infof("teardown complete") return nil } diff --git a/server/server_test.go b/server/server_test.go index 5e2a47dfa..a449d5838 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -531,6 +531,14 @@ func TestClusteringNodesReplica1(t *testing.T) { } } +func TestUpAndDown(t *testing.T) { + c := test.NewCommandNode(t) + err := c.UpAndDown() + if err != nil { + t.Fatalf("server up-and-down: %v", err) + } +} + func TestClusteringNodesReplica2(t *testing.T) { // Because this test shuts down 2 nodes, it needs to start as a 5-node // cluster in order to retain enough available nodes for raft leader diff --git a/test/pilosa.go b/test/pilosa.go index 925213cd1..d1deb491e 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -60,8 +60,6 @@ func newCommand(tb testing.TB, opts ...server.CommandOption) *Command { // a problem with PDK tests which used pilosa/client as well. We put it at the // beginning of the option slice so that it can be overridden by user-passed // options. - // Also set TranslateFile MapSize to a smaller number so memory allocation - // does not fail on 32-bit systems. opts = append([]server.CommandOption{ server.OptCommandCloseTimeout(time.Millisecond * 2), }, opts...) From 1cc6a87d2077c52b951ea18ed55edd9ddccf7a14 Mon Sep 17 00:00:00 2001 From: reesporte Date: Tue, 16 Nov 2021 14:53:30 -0600 Subject: [PATCH 143/222] refactor and add test --- server/pg.go | 2 +- util.go | 11 +++++++++-- util_test.go | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/server/pg.go b/server/pg.go index da6dfd633..169f2e747 100644 --- a/server/pg.go +++ b/server/pg.go @@ -332,7 +332,7 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error switch { case g.Value != nil: if g.FieldOptions.Type == pilosa.FieldTypeTimestamp { - v = time.Unix(0, (int64(*g.Value)+int64(g.FieldOptions.Base))*pilosa.TimeUnitNanos(g.FieldOptions.TimeUnit)).UTC().Format(time.RFC3339Nano) + v = pilosa.FormatTimestampNano(int64(*g.Value), g.FieldOptions.Base, g.FieldOptions.TimeUnit) } else { v = strconv.FormatInt(*g.Value, 10) } diff --git a/util.go b/util.go index 6d4b8e09e..1db0c74fc 100644 --- a/util.go +++ b/util.go @@ -115,8 +115,9 @@ func roaringFragmentHasData(path string, index, field, view string, shard uint64 return } -// GetLoopProgress returns the estimated remaining time to iterate through some items -// as well as the loop completion percentage with the following parameters: +// GetLoopProgress returns the estimated remaining time to iterate through some +// items as well as the loop completion percentage with the following +// parameters: // the start time, the current time, the iteration, and the number of items func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) (remaining time.Duration, pctDone float64) { itemsLeft := total - (iteration + 1) @@ -124,3 +125,9 @@ func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) pctDone = (float64(iteration+1) / float64(total)) * 100 return time.Duration(avgItemTime * float64(itemsLeft)), pctDone } + +// FormatTimestampNano returns the string representation of a timestamp given: +// an epoch value, base, and time unit +func FormatTimestampNano(value, base int64, timeUnit string) string { + return time.Unix(0, (value+base)*TimeUnitNanos(timeUnit)).UTC().Format(time.RFC3339Nano) +} diff --git a/util_test.go b/util_test.go index c659053c5..14ed78c36 100644 --- a/util_test.go +++ b/util_test.go @@ -85,3 +85,21 @@ func TestGetLoopProgress(t *testing.T) { }) } } + +func TestFormatTimestampNano(t *testing.T) { + if FormatTimestampNano(0, 69, "s") != "1970-01-01T00:01:09Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(0, 420, "ms") != "1970-01-01T00:00:00.42Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(420, 0, "μs") != "1970-01-01T00:00:00.00000042Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(420, 69, "us") != "1970-01-01T00:00:00.000489Z" { + t.Fatal("Timestamp not formatted properly") + } + if FormatTimestampNano(69, 420, "ns") != "1970-01-01T00:00:00.000000489Z" { + t.Fatal("Timestamp not formatted properly") + } +} From f8e93871c036802c090d72640b4d4d5a3fd30792 Mon Sep 17 00:00:00 2001 From: reesporte Date: Wed, 17 Nov 2021 08:51:28 -0600 Subject: [PATCH 144/222] refactor safeCopy to pure function and add unit test --- executor.go | 4 ++-- executor_internal_test.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 686379609..3668f9279 100644 --- a/executor.go +++ b/executor.go @@ -254,7 +254,7 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar } } // Must copy out of Tx data before Commiting, because it will become invalid afterwards. - respSafeNoTxData := e.safeCopy(resp) + respSafeNoTxData := safeCopy(resp) // Commit transactions if writing; else let the defer grp.Abort do the rollbacks. if needWriteTxn { @@ -267,7 +267,7 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar // safeCopy copies everything in resp that has Bitmap material, // to avoid anything coming from the mmap-ed Tx storage. -func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { +func safeCopy(resp QueryResponse) (out QueryResponse) { out = QueryResponse{ Err: resp.Err, // error Profile: resp.Profile, // *tracing.Profile diff --git a/executor_internal_test.go b/executor_internal_test.go index b752957ef..ce860c46c 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -475,3 +475,15 @@ func TestGetSorter(t *testing.T) { }) } } + +func TestExecutorSafeCopyDistinctTimestamp(t *testing.T) { + result := DistinctTimestamp{Values: []string{"test", "test"}, Name: "test"} + results := make([]interface{}, 1) + results[0] = result + + response := QueryResponse{Results: results, Err: nil, Profile: nil} + copied := safeCopy(response) + if !reflect.DeepEqual(copied.Results, response.Results) { + t.Fatalf("Did not copy results. got %+v, want %+v", copied.Results, response.Results) + } +} From 6419a87797bcce03261eb7b2a85e110591d5c9da Mon Sep 17 00:00:00 2001 From: reesporte Date: Wed, 17 Nov 2021 09:07:08 -0600 Subject: [PATCH 145/222] use util function for formatting timestamp --- executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor.go b/executor.go index 3668f9279..0d738f72b 100644 --- a/executor.go +++ b/executor.go @@ -1544,7 +1544,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str } results := make([]string, len(r.Pos.Columns())) for i, val := range r.Pos.Columns() { - results[i] = time.Unix(0, (int64(val)+int64(bsig.Base))*TimeUnitNanos(field.options.TimeUnit)).UTC().Format(time.RFC3339Nano) + results[i] = FormatTimeStampNano(int64(val), bsig.Base, field.options.TimeUnit) } return DistinctTimestamp{Name: fieldName, Values: results}, nil } From ea96c10114915b91cb30ef3d7dc024afb364e477 Mon Sep 17 00:00:00 2001 From: reesporte Date: Wed, 17 Nov 2021 09:10:50 -0600 Subject: [PATCH 146/222] fix typo --- executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor.go b/executor.go index 0d738f72b..74e565aed 100644 --- a/executor.go +++ b/executor.go @@ -1544,7 +1544,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str } results := make([]string, len(r.Pos.Columns())) for i, val := range r.Pos.Columns() { - results[i] = FormatTimeStampNano(int64(val), bsig.Base, field.options.TimeUnit) + results[i] = FormatTimestampNano(int64(val), bsig.Base, field.options.TimeUnit) } return DistinctTimestamp{Name: fieldName, Values: results}, nil } From 9302775fd86d027839c10f7f3a22b64e10428385 Mon Sep 17 00:00:00 2001 From: "Kasey C. Rodgers" <49999391+kcrodgers24@users.noreply.github.com> Date: Wed, 17 Nov 2021 08:21:57 -0800 Subject: [PATCH 147/222] generate csv files that simulate Samsung's data --- .gitlab/simulacraData.go | 238 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 .gitlab/simulacraData.go diff --git a/.gitlab/simulacraData.go b/.gitlab/simulacraData.go new file mode 100644 index 000000000..db4fd2012 --- /dev/null +++ b/.gitlab/simulacraData.go @@ -0,0 +1,238 @@ +// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 +// INPUT: none +// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) + +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "strconv" +) + +var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", + "ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS", + "BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE", + "CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE", + "DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK", + "FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD", + "GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL", + "IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR", + "KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO", + "MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ", + "MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL", + "OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT", + "REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM", + "SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL", + "TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT", + "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE", +} + +const totalRecords int = 200000000 + +func main() { + ageField() + ipField() + arbIdField() + optInField() + countryField() + timeField() +} + +func ageField() { + csvFile, err := os.Create("age.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(100) <= 20 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(100)) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(100)) + "\n") + } + } + } + writer.Flush() + csvFile.Close() + +} + +func ipField() { + csvFile, err := os.Create("ip.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func arbIdField() { + csvFile, err := os.Create("identifier.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func timeField() { + csvFile, err := os.Create("time.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 4000000 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + value := distro.Uint64() + urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) + + date := generateDate() + + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + + if rand.Intn(5) == 1 { + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + if rand.Intn(10) == 1 { + value := distro.Uint64() + urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + } + + writer.Flush() + csvFile.Close() + +} + +func optInField() { + csvFile, err := os.Create("optin.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + var optString string + + for i := 0; i < totalRecords; i++ { + + if rand.Intn(20) == 1 { //odds that optRandom == 1 are approx 5% + optString = "1" + } else { + optString = "0" + } + + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if rand.Intn(10) == 1 { + if optString == "0" { + optString = "1" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } else { + optString = "0" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } + } + } + + writer.Flush() + csvFile.Close() + +} + +func countryField() { + csvFile, err := os.Create("country.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + //populate countries + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 245 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(2) == 1 { + value := distro.Uint64() + country := countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + + if rand.Intn(10) == 1 { + value = distro.Uint64() + country = countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + } + } + + } + + writer.Flush() + csvFile.Close() + +} + +func generateDate() string { + // generates a random date in format YYYY-MM-DDT00:00 + // does not include leap days or populata the hour/minute time + year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 + month := rand.Intn(12) + 1 + day := rand.Intn(30) + 1 + + if month == 2 && day >= 29 { + day = 28 + } + if month == 4 && day == 31 { + day = 30 + } + if month == 6 && day == 31 { + day = 30 + } + if month == 11 && day == 31 { + day = 30 + } + + monthFix := fmt.Sprintf("%02d", month) + dayFix := fmt.Sprintf("%02d", day) + YMDstring := year + "-" + monthFix + "-" + dayFix + "T00:00" + + return YMDstring +} From a38c219cf38180dbef9a61002c4f00bea7ce5e82 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 11 Nov 2021 13:54:57 -0600 Subject: [PATCH 148/222] Add test for RBF failures This test case triggers a failure in RBF, it's a separate patch to make it easier to see the failure. --- rbf/tx_test.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/rbf/tx_test.go b/rbf/tx_test.go index f7cd78d6a..05cd76df0 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -219,6 +219,96 @@ func TestTx_DeleteBitmap(t *testing.T) { } } +// deallocateTree had a bug which caused a page to be marked neither free +// nor in-use. +func TestTx_DeallocateTree(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer tx.Rollback() + var err error + + // Create bitmap & add value. + if err = tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + const N = 315 + slots := make([]uint64, N) + for i := range slots { + slots[i] = uint64(i) << 20 + } + if _, err = tx.Add("x", slots...); err != nil { + t.Fatal(err) + } + if err = tx.Check(); err != nil { + t.Fatalf("check: %v", err) + } + if err = tx.DeleteBitmap("x"); err != nil { + t.Fatal(err) + } else { + if ok, err := tx.Contains("x", 0); err != nil { + t.Fatal(err) + } else if ok { + t.Fatal("expected no value in recreated bitmap") + } + } + if err = tx.Check(); err != nil { + t.Fatalf("check: %v", err) + } +} + +func TestTx_RecreateBitmap(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer tx.Rollback() + + // Create bitmap & add value. + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + const N = 825000 + slots := make([]uint64, N) + for i := range slots { + slots[i] = uint64(i) << 20 + } + if _, err := tx.Add("x", slots...); err != nil { + t.Fatal(err) + } + err := tx.Commit() + if err != nil { + t.Fatal(err) + } + tx = MustBegin(t, db, true) + defer tx.Rollback() + // Delete bitmap, verifying that it's gone. + if err := tx.DeleteBitmap("x"); err != nil { + t.Fatal(err) + } else { + if ok, err := tx.Contains("x", 0); err != nil { + t.Fatal(err) + } else if ok { + t.Fatal("expected no value in recreated bitmap") + } + } + err = tx.Commit() + if err != nil { + t.Fatal(err) + } + tx = MustBegin(t, db, true) + defer tx.Rollback() + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + if _, err := tx.Add("x", slots...); err != nil { + t.Fatal(err) + } + err = tx.Commit() + if err != nil { + t.Fatal(err) + } +} + func TestTx_RenameBitmap(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) From 9c0c0ec8c135f44d9c0593aa449a34db9a9e51fa Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 10 Nov 2021 17:27:12 -0600 Subject: [PATCH 149/222] There are two related bugs here. First, it is possible for us to end up allocating *or freeing* pages during a modification of the free list, in a way such that the change to the free list means that when we finish the modification which caused the allocate or free, we've overwritten the inner change. Second, when deallocating trees, we don't actually deallocate the branch nodes themselves. The former causes potentially severe data corruption. The latter causes us to gradually leak pages in a way that we don't notice because we only run those tests during the RBF tests. The fix for this is surprisingly intricate, because of the counterintuitive fact that *allocating* a page means *removing* things from the free list (and thus potentially deallocating free list pages), while *freeing* a page means *adding* things to the free list (and thus potentially needing to allocate pages for the free list). While modifying the free list, any allocations we need always just come from the end of the file; we don't try to reuse free pages. If a page becomes *deallocated* by a free list modification, we don't annotate it in the free list at the instant that it happens; we stash that information until the current modification of the free list happens, then iterate through any such pages. I am pretty sure there's virtually never more than one, and I don't actually know that I can create a case wherein we'd end up with the nested case firing, wherein removing a page from the free list causes us to remove another page, but I think if the free list got large and cluttered and needed rebalancing or something it could maybe happen. --- rbf/tx.go | 93 +++++++++++++++++++++++++++++++++++++++++++++++--- rbf/tx_test.go | 89 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 171 insertions(+), 11 deletions(-) diff --git a/rbf/tx.go b/rbf/tx.go index 9ecbe9a02..812d35738 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -56,6 +56,27 @@ type Tx struct { // behavior where an existing container has all its bits cleared // but still sticks around in the database. DeleteEmptyContainer bool + + // It is possible for a modification of the free list to cause a page to + // be allocated or deallocated, which would modify the free list. + // + // For the case where pages need to be allocated during free list + // changes, we can trivially just allocate new pages and not use the + // free list. That's the simple case... + modifyingFreelist bool + // But removals can't be deferred/not-done like that. If a free list + // change causes us to deallocate a page (such as if we're *allocating* + // a page, which causes it to be *removed* from the free list), we really + // do need to record that, but if we try to do it during the update + // process, things could go horribly wrong. So, we have a transient list + // of page numbers which have been deallocated, but it happened *during* + // the modification of the free list. The top-level modification then + // processes them on its way out, using a defer. During the processing + // of this list, we *still* have the flag set, and we make a new list + // while processing the list, so if somehow a pending add to the list + // manages to trigger a *deallocation* (which I don't think should be + // happening), we'll process that one after the current list is processed. + pendingFreelistAdds []uint32 } func (tx *Tx) DBPath() string { @@ -903,16 +924,67 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er } } +// freelistCleanup handles things which we need to add to the free list, +// because they became free during the process of modifying the free list. +// It also marks us as done modifying the free list. Expected usage is that +// you set modifyingFreelist to true, then defer this. +// +// if you are modifying the free list, we can't further change the free +// list during that modification. for page allocations, we can just skip +// the free list check. for deallocations, though, we do need to mark them +// as freed at some point. so, if we're modifying the free list when +// a new freePgno happens, we stash the new pages in here, then apply +// them afterwards. so far as i know, this can actually only happen +// during an allocate, when we're removing entries from the free list, and +// the add path doesn't ever trigger it. so, when we remove entries from +// the free list, it's possible that doing so frees up pages that were +// part of the free list, and we then add them. but we don't have to worry +// about that removing things from the free list, because the add logic +// already just uses new pages rather than trying to use the free list +// when it knows the free list is involved. +func (tx *Tx) freelistCleanup(outErr *error) { + defer func() { + // no matter what, we're done with this after this, but we still + // want it set *while* we do this so nothing we do will have side + // effects that collide with what we're doing. + tx.modifyingFreelist = false + }() + if len(tx.pendingFreelistAdds) == 0 { + return + } + c := Cursor{tx: tx} + c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} + for len(tx.pendingFreelistAdds) > 0 { + var pass []uint32 + pass, tx.pendingFreelistAdds = tx.pendingFreelistAdds, nil + for _, pgno := range pass { + if changed, err := c.Add(uint64(pgno)); err != nil { + if outErr != nil && *outErr == nil { + *outErr = err + } + return + } else if !changed { + PanicOn(fmt.Sprintf("rbf.Tx.freePgno(): double free: %d", tx.pendingFreelistAdds)) + } + } + } +} + // allocatePgno returns a page number for a new available page. This page may be // pulled from the free list or, if no free pages are available, it will be // created by extending the file size. -func (tx *Tx) allocatePgno() (uint32, error) { +func (tx *Tx) allocatePgno() (_ uint32, outErr error) { + if tx.modifyingFreelist { + return tx.allocateNewPgno(), nil + } // Attempt to find page in freelist. pgno, err := tx.nextFreelistPageNo() if err != nil { return 0, err } else if pgno != 0 { + tx.modifyingFreelist = true + defer tx.freelistCleanup(&outErr) c := Cursor{tx: tx} c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} if changed, err := c.Remove(uint64(pgno)); err != nil { @@ -922,11 +994,16 @@ func (tx *Tx) allocatePgno() (uint32, error) { } return pgno, nil } + // no freelist pages, fall back + return tx.allocateNewPgno(), nil +} +// allocateNewPgno requests a new page unconditionally, ignoring the free list. +func (tx *Tx) allocateNewPgno() uint32 { // Increment the total page count by one and return the last page. - pgno = readMetaPageN(tx.meta[:]) + pgno := readMetaPageN(tx.meta[:]) writeMetaPageN(tx.meta[:], pgno+1) - return pgno, nil + return pgno } func (tx *Tx) nextFreelistPageNo() (uint32, error) { @@ -952,10 +1029,16 @@ func (tx *Tx) nextFreelistPageNo() (uint32, error) { } // deallocate releases a page number to the freelist. -func (tx *Tx) freePgno(pgno uint32) error { +func (tx *Tx) freePgno(pgno uint32) (outErr error) { + if tx.modifyingFreelist { + tx.pendingFreelistAdds = append(tx.pendingFreelistAdds, pgno) + return nil + } c := Cursor{tx: tx} c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])} + tx.modifyingFreelist = true + defer tx.freelistCleanup(&outErr) if changed, err := c.Add(uint64(pgno)); err != nil { return err } else if !changed { @@ -979,7 +1062,7 @@ func (tx *Tx) deallocateTree(pgno uint32) error { return err } } - return nil + return tx.freePgno(pgno) case PageTypeLeaf: return tx.freePgno(pgno) diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 05cd76df0..e4b68f4b6 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/molecula/featurebase/v2/rbf" + "github.com/molecula/featurebase/v2/roaring" ) func TestTx_CommitRollback(t *testing.T) { @@ -245,12 +246,11 @@ func TestTx_DeallocateTree(t *testing.T) { } if err = tx.DeleteBitmap("x"); err != nil { t.Fatal(err) - } else { - if ok, err := tx.Contains("x", 0); err != nil { - t.Fatal(err) - } else if ok { - t.Fatal("expected no value in recreated bitmap") - } + } + if ok, err := tx.Contains("x", 0); err != nil { + t.Fatal(err) + } else if ok { + t.Fatal("expected no value in recreated bitmap") } if err = tx.Check(); err != nil { t.Fatalf("check: %v", err) @@ -369,6 +369,83 @@ func TestTx_Add_Quick(t *testing.T) { }) } +func TestTx_DeallocateToFreeList(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer tx.Rollback() + var err error + + // Create bitmap & add value. + if err = tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + if err = tx.CreateBitmap("y"); err != nil { + t.Fatal(err) + } + const N = 12274831 + slots := make([]uint64, N) + for i := range slots { + slots[i] = uint64(i) << 10 + } + bm := roaring.NewBitmap(slots...) + if _, err = tx.AddRoaring("x", bm); err != nil { + t.Fatal(err) + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + for i := 0; i < 500; i++ { + if _, err := tx.Add("y", uint64(i)<<16); err != nil { + t.Fatal(err) + } + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + if err := tx.DeleteBitmap("y"); err != nil { + t.Fatal(err) + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + tx = MustBegin(t, db, true) + defer tx.Rollback() + // Delete bitmap, verifying that it's gone. + if err := tx.DeleteBitmap("x"); err != nil { + t.Fatal(err) + } else { + if ok, err := tx.Contains("x", 0); err != nil { + t.Fatal(err) + } else if ok { + t.Fatal("expected no value in recreated bitmap") + } + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + tx = MustBegin(t, db, true) + defer tx.Rollback() + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + if _, err := tx.AddRoaring("x", bm); err != nil { + t.Fatal(err) + } + if err = tx.Check(); err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } +} + func TestTx_AddRemove_Quick(t *testing.T) { if testing.Short() { t.Skip("-short enabled, skipping") From 442d109814ab58fd108b9d42bca4fd760ba798bd Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 18 Nov 2021 10:28:49 -0600 Subject: [PATCH 150/222] created qa folder --- .gitlab/.gitlab-ci.yml | 6 +++--- {.gitlab => qa/scripts}/cloud-init.sh | 0 {.gitlab => qa/scripts}/configureFeatureBase.json | 0 {.gitlab => qa/scripts}/featurebase.conf | 0 {.gitlab => qa/scripts}/featurebase.service | 0 {.gitlab => qa/scripts}/ingestWorkload.sh | 0 {.gitlab => qa/simulacraData}/simulacraData.go | 0 7 files changed, 3 insertions(+), 3 deletions(-) rename {.gitlab => qa/scripts}/cloud-init.sh (100%) rename {.gitlab => qa/scripts}/configureFeatureBase.json (100%) rename {.gitlab => qa/scripts}/featurebase.conf (100%) rename {.gitlab => qa/scripts}/featurebase.service (100%) rename {.gitlab => qa/scripts}/ingestWorkload.sh (100%) rename {.gitlab => qa/simulacraData}/simulacraData.go (100%) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ccd710889..82710bb4d 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -225,15 +225,15 @@ deploy node for linux amd64: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) - echo $INSTANCE_ID > linux-amd64-instance.txt - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec3-user@$PUBLIC_IP:. + - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: - job: build for linux amd64 artifacts: diff --git a/.gitlab/cloud-init.sh b/qa/scripts/cloud-init.sh similarity index 100% rename from .gitlab/cloud-init.sh rename to qa/scripts/cloud-init.sh diff --git a/.gitlab/configureFeatureBase.json b/qa/scripts/configureFeatureBase.json similarity index 100% rename from .gitlab/configureFeatureBase.json rename to qa/scripts/configureFeatureBase.json diff --git a/.gitlab/featurebase.conf b/qa/scripts/featurebase.conf similarity index 100% rename from .gitlab/featurebase.conf rename to qa/scripts/featurebase.conf diff --git a/.gitlab/featurebase.service b/qa/scripts/featurebase.service similarity index 100% rename from .gitlab/featurebase.service rename to qa/scripts/featurebase.service diff --git a/.gitlab/ingestWorkload.sh b/qa/scripts/ingestWorkload.sh similarity index 100% rename from .gitlab/ingestWorkload.sh rename to qa/scripts/ingestWorkload.sh diff --git a/.gitlab/simulacraData.go b/qa/simulacraData/simulacraData.go similarity index 100% rename from .gitlab/simulacraData.go rename to qa/simulacraData/simulacraData.go From eb717f568dcee4f7801655d9ace3029158b4df28 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 18 Nov 2021 11:49:38 -0600 Subject: [PATCH 151/222] fix for ec2 ip filtering --- .gitlab/.gitlab-ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index ccd710889..385e66eab 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -221,16 +221,17 @@ deploy node for linux amd64: - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt install jq script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm - - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - - PUBLIC_IP=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - - echo $PUBLIC_IP - - INSTANCE_ID=$(aws ec2 describe-instances --filters 'Name=tag:Name, Values=linux-amd64-node' 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text --profile $PROFILE) + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json + - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json) - echo $INSTANCE_ID > linux-amd64-instance.txt + - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data + - PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + - echo $PUBLIC_IP - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 From d40eefb36499f2bf4a26d5ccd911092faec799e2 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 18 Nov 2021 12:06:07 -0600 Subject: [PATCH 152/222] added host:port as a command line input --- qa/scripts/ingestWorkload.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/qa/scripts/ingestWorkload.sh b/qa/scripts/ingestWorkload.sh index 2502dba2a..23d11dd32 100755 --- a/qa/scripts/ingestWorkload.sh +++ b/qa/scripts/ingestWorkload.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # To run: -# ./ingestWorkload.sh {Path for featurebase binary} {Path for directory with csv files} {initialize flag} +# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag} function delete_field { if (($INITIALIZE == 0)); @@ -34,6 +34,10 @@ function ingest_set_field { FEATUREBASE_PATH=$1 shift +# featurebase host & port +HOST=$1 +shift + # path for directory with csv directory files for all fields to be ingested CSV_DIR_PATH=$1 shift @@ -45,8 +49,6 @@ shift # get a list of csv files in the directory CSV_FILES=`ls $CSV_DIR_PATH/*.csv` -# featurebase host -HOST="localhost:10101" # assign index name INDEX="samsung" if (($INITIALIZE == 1)); @@ -62,24 +64,19 @@ for CSV_FILE in ${CSV_FILES[@]} if [[ "$FIELD" == *"age"* ]]; then MAX=100 - echo $CSV_FILE $MAX ingest_int_field elif [[ "$FIELD" == *"identifier"* ]]; then MAX=$((2**63 - 1)) # compute max value for 64bit - echo $CSV_FILE $MAX ingest_int_field elif [[ "$FIELD" == *"ip"* ]]; then MAX=$((2**31 - 1)) # compute max value for 32bit - echo $CSV_FILE $MAX ingest_int_field elif [[ "$FIELD" == *"time"* ]]; then - echo $CSV_FILE ingest_time_field else - echo $CSV_FILE ingest_set_field fi done From 5d489198bde46044cb265719dcdd3deb15426e4a Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 18 Nov 2021 12:11:48 -0600 Subject: [PATCH 153/222] fixed apt install bug --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 385e66eab..5ab7d3216 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -221,7 +221,7 @@ deploy node for linux amd64: - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt install jq + - apt update && apt -y install jq script: - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) From 4e3f31471cee66357b3c2d141952b3c2ecdd7484 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 18 Nov 2021 12:42:09 -0600 Subject: [PATCH 154/222] remove double quotes from variable name --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 5ab7d3216..0dda19409 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -227,7 +227,7 @@ deploy node for linux amd64: - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json) + - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - echo $INSTANCE_ID > linux-amd64-instance.txt - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) From 14050c567c8f535da21edaa77abaa928849cd587 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Thu, 18 Nov 2021 13:15:44 -0800 Subject: [PATCH 155/222] corrects naming convention and age range --- qa/simulacraData/simulacra_data.go | 238 +++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 qa/simulacraData/simulacra_data.go diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go new file mode 100644 index 000000000..e09769a74 --- /dev/null +++ b/qa/simulacraData/simulacra_data.go @@ -0,0 +1,238 @@ +// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 +// INPUT: none +// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) + +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "strconv" +) + +var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", + "ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS", + "BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE", + "CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE", + "DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK", + "FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD", + "GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL", + "IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR", + "KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO", + "MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ", + "MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL", + "OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT", + "REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM", + "SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL", + "TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT", + "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE", +} + +const totalRecords int = 200000000 + +func main() { + ageField() + ipField() + arbIdField() + optInField() + countryField() + timeField() +} + +func ageField() { + csvFile, err := os.Create("age.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(100) <= 20 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + } + } + } + writer.Flush() + csvFile.Close() + +} + +func ipField() { + csvFile, err := os.Create("ip.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func arbIdField() { + csvFile, err := os.Create("identifier.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + for i := 0; i < totalRecords; i++ { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if rand.Intn(10) == 1 { + writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + } + } + + writer.Flush() + csvFile.Close() + +} + +func timeField() { + csvFile, err := os.Create("time.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 4000000 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + value := distro.Uint64() + urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) + + date := generateDate() + + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + + if rand.Intn(5) == 1 { + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + if rand.Intn(10) == 1 { + value := distro.Uint64() + urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) + date = generateDate() + writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + } + + } + + writer.Flush() + csvFile.Close() + +} + +func optInField() { + csvFile, err := os.Create("optin.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + var optString string + + for i := 0; i < totalRecords; i++ { + + if rand.Intn(20) == 1 { //odds that optRandom == 1 are approx 5% + optString = "1" + } else { + optString = "0" + } + + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if rand.Intn(10) == 1 { + if optString == "0" { + optString = "1" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } else { + optString = "0" + writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + } + } + } + + writer.Flush() + csvFile.Close() + +} + +func countryField() { + csvFile, err := os.Create("country.csv") + if err != nil { + log.Fatalf("failed to create file %s", err) + } + writer := bufio.NewWriter(csvFile) + + //populate countries + r := rand.New(rand.NewSource(rand.Int63())) + s := 1.01 + v := 1.01 + var imax uint64 = 245 + + distro := rand.NewZipf(r, s, v, imax) + + for i := 0; i < totalRecords; i++ { + if rand.Intn(2) == 1 { + value := distro.Uint64() + country := countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + + if rand.Intn(10) == 1 { + value = distro.Uint64() + country = countryList[value] + writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + } + } + + } + + writer.Flush() + csvFile.Close() + +} + +func generateDate() string { + // generates a random date in format YYYY-MM-DDT00:00 + // does not include leap days or populata the hour/minute time + year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 + month := rand.Intn(12) + 1 + day := rand.Intn(30) + 1 + + if month == 2 && day >= 29 { + day = 28 + } + if month == 4 && day == 31 { + day = 30 + } + if month == 6 && day == 31 { + day = 30 + } + if month == 11 && day == 31 { + day = 30 + } + + monthFix := fmt.Sprintf("%02d", month) + dayFix := fmt.Sprintf("%02d", day) + YMDstring := year + "-" + monthFix + "-" + dayFix + "T00:00" + + return YMDstring +} From 90ace6b970c3e5cc3d450530c00eeff6bd110365 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Thu, 18 Nov 2021 13:47:23 -0800 Subject: [PATCH 156/222] deletes duplicate file and adds license header --- qa/simulacraData/simulacraData.go | 238 ----------------------------- qa/simulacraData/simulacra_data.go | 14 ++ 2 files changed, 14 insertions(+), 238 deletions(-) delete mode 100644 qa/simulacraData/simulacraData.go diff --git a/qa/simulacraData/simulacraData.go b/qa/simulacraData/simulacraData.go deleted file mode 100644 index db4fd2012..000000000 --- a/qa/simulacraData/simulacraData.go +++ /dev/null @@ -1,238 +0,0 @@ -// PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 -// INPUT: none -// OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) - -package main - -import ( - "bufio" - "fmt" - "log" - "math/rand" - "os" - "strconv" -) - -var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", - "ARM", "ASM", "ATA", "ATF", "ATG", "AUS", "AUT", "AZE", "BDI", "BEL", "BEN", "BFA", "BGD", "BGR", "BHR", "BHS", - "BIH", "BLM", "BLR", "BLZ", "BMU", "BOL", "BRA", "BRB", "BRN", "BTN", "BVT", "BWA", "CAF", "CAN", "CCK", "CHE", - "CHL", "CHN", "CIV", "CMR", "COD", "COG", "COK", "COL", "COM", "CPV", "CRI", "CUB", "CXR", "CYM", "CYP", "CZE", - "DEU", "DJI", "DMA", "DNK", "DOM", "DZA", "ECU", "EGY", "ERI", "ESH", "ESP", "EST", "ETH", "FIN", "FJI", "FLK", - "FRA", "FRO", "FSM", "GAB", "GBR", "GEO", "GGY", "GHA", "GIB", "GIN", "GLP", "GMB", "GNB", "GNQ", "GRC", "GRD", - "GRL", "GTM", "GUF", "GUM", "GUY", "HKG", "HMD", "HND", "HRV", "HTI", "HUN", "IDN", "IMN", "IND", "IOT", "IRL", - "IRN", "IRQ", "ISL", "ISR", "ITA", "JAM", "JEY", "JOR", "JPN", "KAZ", "KEN", "KGZ", "KHM", "KIR", "KNA", "KOR", - "KWT", "LAO", "LBN", "LBR", "LBY", "LCA", "LIE", "LKA", "LSO", "LTU", "LUX", "LVA", "MAC", "MAF", "MAR", "MCO", - "MDA", "MDG", "MDV", "MEX", "MHL", "MKD", "MLI", "MLT", "MMR", "MNE", "MNG", "MNP", "MOZ", "MRT", "MSR", "MTQ", - "MUS", "MWI", "MYS", "MYT", "NAM", "NCL", "NER", "NFK", "NGA", "NIC", "NIU", "NLD", "NOR", "NPL", "NRU", "NZL", - "OMN", "PAK", "PAN", "PCN", "PER", "PHL", "PLW", "PNG", "POL", "PRI", "PRK", "PRT", "PRY", "PSE", "PYF", "QAT", - "REU", "ROU", "RUS", "RWA", "SAU", "SDN", "SEN", "SGP", "SGS", "SHN", "SJM", "SLB", "SLE", "SLV", "SMR", "SOM", - "SPM", "SRB", "STP", "SUR", "SVK", "SVN", "SWE", "SWZ", "SYC", "SYR", "TCA", "TCD", "TGO", "THA", "TJK", "TKL", - "TKM", "TLS", "TON", "TTO", "TUN", "TUR", "TUV", "TWN", "TZA", "UGA", "UKR", "UMI", "URY", "USA", "UZB", "VAT", - "VCT", "VEN", "VGB", "VIR", "VNM", "VUT", "WLF", "WSM", "YEM", "ZAF", "ZMB", "ZWE", -} - -const totalRecords int = 200000000 - -func main() { - ageField() - ipField() - arbIdField() - optInField() - countryField() - timeField() -} - -func ageField() { - csvFile, err := os.Create("age.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - for i := 0; i < totalRecords; i++ { - if rand.Intn(100) <= 20 { - writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(100)) + "\n") - if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(100)) + "\n") - } - } - } - writer.Flush() - csvFile.Close() - -} - -func ipField() { - csvFile, err := os.Create("ip.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - for i := 0; i < totalRecords; i++ { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") - if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") - } - } - - writer.Flush() - csvFile.Close() - -} - -func arbIdField() { - csvFile, err := os.Create("identifier.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - for i := 0; i < totalRecords; i++ { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") - if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") - } - } - - writer.Flush() - csvFile.Close() - -} - -func timeField() { - csvFile, err := os.Create("time.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - r := rand.New(rand.NewSource(rand.Int63())) - s := 1.01 - v := 1.01 - var imax uint64 = 4000000 - - distro := rand.NewZipf(r, s, v, imax) - - for i := 0; i < totalRecords; i++ { - value := distro.Uint64() - urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) - - date := generateDate() - - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") - - if rand.Intn(5) == 1 { - date = generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") - } - - if rand.Intn(10) == 1 { - value := distro.Uint64() - urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) - date = generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") - } - - } - - writer.Flush() - csvFile.Close() - -} - -func optInField() { - csvFile, err := os.Create("optin.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - var optString string - - for i := 0; i < totalRecords; i++ { - - if rand.Intn(20) == 1 { //odds that optRandom == 1 are approx 5% - optString = "1" - } else { - optString = "0" - } - - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") - if rand.Intn(10) == 1 { - if optString == "0" { - optString = "1" - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") - } else { - optString = "0" - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") - } - } - } - - writer.Flush() - csvFile.Close() - -} - -func countryField() { - csvFile, err := os.Create("country.csv") - if err != nil { - log.Fatalf("failed to create file %s", err) - } - writer := bufio.NewWriter(csvFile) - - //populate countries - r := rand.New(rand.NewSource(rand.Int63())) - s := 1.01 - v := 1.01 - var imax uint64 = 245 - - distro := rand.NewZipf(r, s, v, imax) - - for i := 0; i < totalRecords; i++ { - if rand.Intn(2) == 1 { - value := distro.Uint64() - country := countryList[value] - writer.WriteString(country + "," + strconv.Itoa(i) + "\n") - - if rand.Intn(10) == 1 { - value = distro.Uint64() - country = countryList[value] - writer.WriteString(country + "," + strconv.Itoa(i) + "\n") - } - } - - } - - writer.Flush() - csvFile.Close() - -} - -func generateDate() string { - // generates a random date in format YYYY-MM-DDT00:00 - // does not include leap days or populata the hour/minute time - year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 - month := rand.Intn(12) + 1 - day := rand.Intn(30) + 1 - - if month == 2 && day >= 29 { - day = 28 - } - if month == 4 && day == 31 { - day = 30 - } - if month == 6 && day == 31 { - day = 30 - } - if month == 11 && day == 31 { - day = 30 - } - - monthFix := fmt.Sprintf("%02d", month) - dayFix := fmt.Sprintf("%02d", day) - YMDstring := year + "-" + monthFix + "-" + dayFix + "T00:00" - - return YMDstring -} diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index e09769a74..6e3dffc7e 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -1,3 +1,17 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // PURPOSE: Generate data for Samsung unique workflow/use case, as described in Jira Ticket FB-971 // INPUT: none // OUTPUT: 6 csv files, containing approx 1 billion lines of data associated to 200 million unique records (approx 28BGB of data) From 87ededd61e55ef88ba7973d050d1ae88679ec705 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 18 Nov 2021 16:23:12 -0600 Subject: [PATCH 157/222] Modified random query to use vegeta library --- .gitlab/cloud-init.sh | 6 +- cmd/random-query/main.go | 276 ++++++++++++++++++++-------------- cmd/random-query/main_test.go | 8 +- go.mod | 1 + go.sum | 41 +++++ 5 files changed, 210 insertions(+), 122 deletions(-) diff --git a/.gitlab/cloud-init.sh b/.gitlab/cloud-init.sh index 055d0f6aa..8d710d0b2 100755 --- a/.gitlab/cloud-init.sh +++ b/.gitlab/cloud-init.sh @@ -14,8 +14,8 @@ echo 'cat /proc/sys/fs/file-max' # yum install golang -y # latest verion in ec2 is 1.15.14 # install go 1.16.9 manually -curl -O https://dl.google.com/go/go1.16.9.linux-amd64.tar.gz -tar xvf go1.16.9.linux-amd64.tar.gz +curl -O https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz +tar xvf go1.16.10.linux-amd64.tar.gz chown -R root:root ./go mv go /usr/local echo "export PATH=/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/ec2-user/.local/bin:/home/ec2-user/bin:/usr/local/go/bin" | tee -a /etc/profile > /dev/null @@ -23,4 +23,4 @@ source /etc/profile # install aws session manager pluggin curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm" -o "session-manager-plugin.rpm" -yum install -y session-manager-plugin.rpm \ No newline at end of file +yum install -y session-manager-plugin.rpm diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 1faafad1e..5c3bc4ee0 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -18,34 +18,47 @@ import ( "context" "flag" "fmt" + "io/ioutil" "math" "math/rand" nethttp "net/http" + "os" "strconv" "strings" "time" - "github.com/molecula/featurebase/v2" + "github.com/gogo/protobuf/proto" + pilosa "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/client" "github.com/molecula/featurebase/v2/http" + "github.com/molecula/featurebase/v2/pb" "github.com/molecula/featurebase/v2/pql" + "github.com/molecula/featurebase/v2/vprint" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck + "github.com/pkg/errors" + vegeta "github.com/tsenart/vegeta/v12/lib" ) // RandomQueryConfig type RandomQueryConfig struct { // user facing flags - HostPort string // -hostport - TreeDepth int // -d - QueryCount int // -n - Verbose bool // -v - VeryVerbose bool // -V + HostPort string // -hostport + TreeDepth int // -d + QueryCount int // -n + Verbose bool // -v + NumRuns int TimeFromArg string // --time.from TimeToArg string // --time.to TimeFrom time.Time // parsed time TimeTo time.Time // parsed time TimeRange int64 // hours between parsed times + Index string + QPS int + SrcFile string + Duration time.Duration + Target vegeta.Target IndexMap map[string]*Features @@ -93,12 +106,16 @@ var defaultStartTime = defaultEndTime.Add(-5 * 365 * 24 * time.Hour) // call DefineFlags before myflags.Parse() func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.") - fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.") - fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.") + fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") + fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") + fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") + fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") + fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") + fs.IntVar(&cfg.QPS, "qps", 10, "number of currernt requests per sec to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") - fs.BoolVar(&cfg.VeryVerbose, "V", false, "show query results") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") + fs.StringVar(&cfg.SrcFile, "query-file", "", "use pql contained in this file for query batch instead of generating") } // call c.ValidateConfig() after myflags.Parse() @@ -144,7 +161,6 @@ func main() { fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err) os.Exit(1) } - err = cfg.Run() if err != nil { @@ -159,76 +175,24 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - ctx := context.Background() - totalQ := 0 - loops := 0 - t0 := time.Now() - - report := func() { - dur := time.Since(t0) - if dur > 0 { - qps := 1e9 * float64(totalQ) / float64(dur) - AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps) - } else { - AlwaysPrintf("totalQueries run: %v elapsed: %v qps: N/A", totalQ, dur) - } - } - defer report() - -NewSetup: err = cfg.Setup(cli) if err != nil { return err } + rate := vegeta.Rate{Freq: cfg.QPS, Per: time.Second} + duration := cfg.Duration + targeter := vegeta.NewStaticTargeter(cfg.Target) + attacker := vegeta.NewAttacker() - if len(cfg.IndexMap) == 0 { - return fmt.Errorf("no rows to query") - } - - var indexes []string - for index := range cfg.IndexMap { - indexes = append(indexes, index) - } - - for j := 0; ; j++ { - if cfg.QueryCount > 0 { - if j >= cfg.QueryCount { - break - } - } else { - // else keep doing queries forever... - if loops > 0 && loops%500 == 0 { - // ...but account for any new data arrived by getting - // the schema and rows again every so often. - loops++ - goto NewSetup - } + for i := 0; i < cfg.NumRuns; i++ { + vprint.VV("================") + var metrics vegeta.Metrics + for res := range attacker.Attack(targeter, rate, duration, "Big Bang!") { + metrics.Add(res) } - if totalQ > 0 && totalQ%100 == 0 { - report() - } - - index := indexes[cfg.Rnd.Intn(len(indexes))] - - pql, err := cfg.GenQuery(index) - PanicOn(err) - - if cfg.Verbose { - fmt.Printf("pql = '%v'\n", pql) - } - - // Query node0. - res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql}) - if err != nil { - AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql) - return err - } - if cfg.VeryVerbose { - fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) - } - totalQ++ - loops++ - + metrics.Close() + rpt := vegeta.NewTextReporter(&metrics) + rpt(os.Stdout) } return nil @@ -238,6 +202,7 @@ type Features struct { Slc []IndexFieldRow Ranges []IndexFieldRange Distinctables []IndexFieldRange + Stores []IndexFieldRow SlcWeight int RangeWeight int } @@ -283,7 +248,7 @@ func (fea *IndexFieldRow) Query(cfg *RandomQueryConfig) *Tree { endTime.Format(pilosaTimeFmt)) } if fea.IsRowKey { - return &Tree{S: fmt.Sprintf("Row(%v='%v'%s)", fea.Field, fea.RowKey, fromTo)} + return &Tree{S: fmt.Sprintf(`Row(%v="%v"%s)`, fea.Field, fea.RowKey, fromTo)} } return &Tree{S: fmt.Sprintf("Row(%v=%v%s)", fea.Field, fea.RowID, fromTo)} } @@ -341,6 +306,9 @@ func (i *IndexFieldRange) Query(cfg *RandomQueryConfig) *Tree { // and spits back a PQL query // func (cfg *RandomQueryConfig) Setup(api API) (err error) { + if cfg.SrcFile != "" { + return cfg.buildPayload() + } ctx := context.Background() cfg.Info, err = api.Schema(ctx) if err != nil { @@ -348,58 +316,114 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { } foundIntField := false for i, ii := range cfg.Info { - _ = i - for k, fld := range ii.Fields { - _ = k - switch fld.Options.Type { - case "set", "mutex", "time": - pql := fmt.Sprintf("Rows(%v)", fld.Name) + if ii.Name == cfg.Index { - res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) - PanicOn(err) - if cfg.VeryVerbose { - fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) - } - // if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers. - // e.g. - // success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"} - // success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}' + _ = i + for k, fld := range ii.Fields { + _ = k + switch fld.Options.Type { + case "set", "mutex", "time": + pql := fmt.Sprintf("Rows(%v)", fld.Name) - switch x := res.Results[0].(type) { - case *pilosa.RowIdentifiers: - // internalClient gets this - cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time") - case pilosa.RowIdentifiers: - // test gets this - cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time") + res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) + PanicOn(err) + switch x := res.Results[0].(type) { + case *pilosa.RowIdentifiers: + cfg.AddResponse(ii.Name, fld.Name, x, fld.Options.Type == "time", fld.Options.Type == "set") + case pilosa.RowIdentifiers: + cfg.AddResponse(ii.Name, fld.Name, &x, fld.Options.Type == "time", fld.Options.Type == "set") + } + case "int": + foundIntField = true + fallthrough + case "decimal": + cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") + default: + AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } - case "int": - foundIntField = true - fallthrough // I bet you thought you'd never see this used - case "decimal": - cfg.AddIntField(ii.Name, fld.Name, fld.Options.Min, fld.Options.Max, fld.Options.Scale, fld.Options.Type == "decimal") - default: - AlwaysPrintf("ignoring field %q: unhandled type %q\n", fld.Name, fld.Options.Type) } } } - cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") } seed := int64(42) cfg.Rnd = rand.New(rand.NewSource(seed)) - - return nil + return cfg.buildPayload() } -func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool) { +func (cfg *RandomQueryConfig) buildPayload() error { + var request strings.Builder + if cfg.SrcFile != "" { + b, err := ioutil.ReadFile(cfg.SrcFile) // just pass the file name + if err != nil { + return err + } + request.WriteString(string(b)) + } else { + + for i := 0; i < cfg.QueryCount; i++ { + pql, err := cfg.GenQuery(cfg.Index) + if err != nil { + return err + } + request.WriteString(pql) + } + } + //TODO (twg) tls support + path := fmt.Sprintf("http://%s/index/%s/query", cfg.HostPort, cfg.Index) + header := nethttp.Header{ + "Content-Type": []string{"application/x-protobuf"}, + "Accept": []string{"application/x-protobuf"}, + "PQL-Version": []string{client.PQLVersion}, + } + + req := &pb.QueryRequest{ + Query: request.String(), + } + vprint.VV("%v", request.String()) + payload, err := proto.Marshal(req) + if err != nil { + return errors.Wrap(err, "marshaling request to protobuf") + } + cfg.Target = vegeta.Target{ + Method: "POST", + URL: path, + Body: payload, + Header: header, + } + return nil +} +func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers, hasTime bool, isSet bool) { + storeID := uint64(0) for _, rowID := range x.Rows { cfg.AddFeature(index, field, rowID, "", false, hasTime) + storeID = rowID } + storeKey := "" for _, rowKey := range x.Keys { cfg.AddFeature(index, field, 0, rowKey, true, hasTime) + storeKey = rowKey + } + idx := cfg.IndexMap[index] + if isSet { + if storeID > 0 { + idx.Stores = append(idx.Stores, IndexFieldRow{ + Index: index, + Field: field, + RowID: storeID + 1, + }) + } + if storeKey != "" { + idx.Stores = append(idx.Stores, IndexFieldRow{ + Index: index, + Field: field, + RowKey: storeKey + "_1", + IsRowKey: true, + }) + + } } } @@ -411,10 +435,12 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } - if min.Scale != scale || max.Scale != scale { - PanicOn(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal", - min.Scale, max.Scale, scale)) - } + /* if min.Scale != scale || max.Scale != scale { + + PanicOn(fmt.Sprintf("scale error; %v:%v min scale %d, max scale %d, field scale %d, assumed they'd be equal", + index, field, min.Scale, max.Scale, scale)) + } + */ effectiveRange := uint64(max.Value) - uint64(min.Value) + 1 // if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which @@ -449,11 +475,32 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci } func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { - tree := cfg.GenTree(index, cfg.TreeDepth) pql = tree.ToPQL() - // avoid using too much bandwidth, just count the final bitmap. + dice := cfg.Rnd.Intn(9) + if dice == 3 { // 1 in 9 of getting a store + idx := cfg.IndexMap[index] + if len(idx.Stores) > 0 { + i := cfg.Rnd.Intn(len(idx.Stores)) + fr := idx.Stores[i] + var key string + if fr.IsRowKey { + key = fmt.Sprintf(`%v="%v"`, fr.Field, fr.RowKey) + c := strings.LastIndex(fr.RowKey, "_") + n, err := strconv.Atoi(fr.RowKey[c+1:]) + PanicOn(err) + n += 1 + fr.RowKey = fmt.Sprintf("%v%v", fr.RowKey[:c+1], n) + } else { + key = fmt.Sprintf("%v=%v", fr.Field, fr.RowID) + fr.RowID = fr.RowID + 1 + } + idx.Stores[i] = fr + pql = fmt.Sprintf("Store(%v,%v)Count(Row(%v))", pql, key, key) + return + } + } pql = fmt.Sprintf("Count(%v)", pql) return } @@ -522,7 +569,6 @@ func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { } func (tr *Tree) ToPQL() (s string) { - if len(tr.Chd) == 0 { // leaf return tr.S diff --git a/cmd/random-query/main_test.go b/cmd/random-query/main_test.go index eb6cc9040..7ad6936fa 100644 --- a/cmd/random-query/main_test.go +++ b/cmd/random-query/main_test.go @@ -20,7 +20,7 @@ import ( "strconv" "testing" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/server" @@ -65,8 +65,8 @@ func Test_RandomQuery(t *testing.T) { ctx := context.Background() - indexes := []string{"rick", "morty"} - fieldName := []string{"f", "flying_car"} + indexes := []string{"rick"} + fieldName := []string{"f"} idx := make([]*pilosa.Index, len(indexes)) field := make([]*pilosa.Field, len(indexes)) @@ -144,7 +144,7 @@ func Test_RandomQuery(t *testing.T) { //qcx.Reset() } // end of setup. - + cfg.Index = indexes[0] PanicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API))) for j := 0; j < 4; j++ { diff --git a/go.mod b/go.mod index 53a0860d3..add6082e3 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 + github.com/tsenart/vegeta/v12 v12.8.4 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.4.0+incompatible // indirect github.com/zeebo/blake3 v0.1.1 diff --git a/go.sum b/go.sum index 0b9624b38..4224f965d 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/jsonschema v0.0.0-20180308105923-f2c93856175a/go.mod h1:qpebaTNSsyUn5rPSJMsfqEtDw71TTggXM6stUDI16HA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -42,8 +43,11 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= +github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -75,6 +79,9 @@ github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051 h1:ByJUvQYyTtNNCVfYNM48q6uYUT4fAlN0wNmd3th4BSo= +github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E= +github.com/dgryski/go-lttb v0.0.0-20180810165845-318fcdf10a77/go.mod h1:Va5MyIzkU0rAM92tn3hb3Anb7oz7KcnixF49+2wOMe4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -120,6 +127,24 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac h1:Q0Jsdxl5jbxouNs1TQYt0gxesYMU4VXRbsTlgDloZ50= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc= +github.com/gonum/diff v0.0.0-20181124234638-500114f11e71 h1:BE6g8oinc3Ek2elIHq+uDOiZgX3/ODi+EerJ48yrrKc= +github.com/gonum/diff v0.0.0-20181124234638-500114f11e71/go.mod h1:22dM4PLscQl+Nzf64qNBurVJvfyvZELT0iRW2l/NN70= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 h1:EvokxLQsaaQjcWVWSV38221VAK7qc2zhaO17bKys/18= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg= +github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2 h1:GUSkTcIe1SlregbHNUKbYDhBsS8lNgYfIp4S4cToUyU= +github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2/go.mod h1:pDgmNM6seYpwvPos3q+zxlXMsbve6mOIPucUnUOrI7Y= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029 h1:8jtTdc+Nfj9AR+0soOeia9UZSvYBvETVHZrugUowJ7M= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029/go.mod h1:Pu4dmpkhSyOzRwuXkOgAvijx4o+4YMUJJo9OvPYMkks= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9 h1:7qnwS9+oeSiOIsiUMajT+0R7HR6hw5NegnKPmn/94oI= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2EAE789SSiSJNqxPaC0aE9J8NTOI0Jo/A= +github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55 h1:Ajwn2ENgC/pKtVat0LEHEWNa4a4VGyYJ1feGSccOzFU= +github.com/gonum/mathext v0.0.0-20181121095525-8a4bf007ea55/go.mod h1:fmo8aiSEWkJeiGXUJf+sPvuDgEFgqIoZSs843ePKrGg= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 h1:V2IgdyerlBa/MxaEFRbV5juy/C3MGdj4ePi+g6ePIp4= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw= +github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b h1:fbskpz/cPqWH8VqkQ7LJghFkl2KPAiIFUHrTJ2O3RGk= +github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b/go.mod h1:Z4GIJBJO3Wa4gD4vbwQxXXZ+WHmW6E9ixmNrwvs0iZs= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -184,6 +209,8 @@ github.com/improbable-eng/grpc-web v0.13.0 h1:7XqtaBWaOCH0cVGKHyvhtcuo6fgW32Y10y github.com/improbable-eng/grpc-web v0.13.0/go.mod h1:6hRR09jOEG81ADP5wCQju1z71g6OL4eEvELdran/3cs= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a h1:vMqgISSVkIqWxCIZs8m1L4096temR7IbYyNdMiBxSPA= +github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9GkyshztGufsdPQWjH+ifgnIr3xNUL5syI70g2dzU1o= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -210,6 +237,8 @@ github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -217,6 +246,7 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.17/go.mod h1:WgzbA6oji13JREwiNsRDNfl7jYdPnmz+VEuLrA+/48M= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -321,6 +351,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A= +github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -338,6 +370,9 @@ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcy github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tsenart/go-tsz v0.0.0-20180814232043-cdeb9e1e981e/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo= +github.com/tsenart/vegeta/v12 v12.8.4 h1:UQ7tG7WkDorKj0wjx78Z4/vsMBP8RJQMGJqRVrkvngg= +github.com/tsenart/vegeta/v12 v12.8.4/go.mod h1:ZiJtwLn/9M4fTPdMY7bdbIeyNeFVE8/AHbWFqCsUuho= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= @@ -367,6 +402,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -416,6 +452,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= @@ -446,6 +483,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -486,6 +524,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -559,6 +598,8 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +pgregory.net/rapid v0.3.3 h1:jCjBsY4ln4Atz78QoBWxUEvAHaFyNDQg9+WU62aCn1U= +pgregory.net/rapid v0.3.3/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 3b92f9493d5e3ce62ff53a91c49579db01e68ad1 Mon Sep 17 00:00:00 2001 From: nm Date: Tue, 9 Nov 2021 04:30:44 +0300 Subject: [PATCH 158/222] add config files for release --- Makefile | 13 +- NOTICE | 15 +- install/featurebase.conf | 373 +++++++++++++++++++++++++++ install/featurebase.debian.service | 13 + install/featurebase.redhat.service | 12 + install/test_installation.Dockerfile | 18 ++ install/test_installation.sh | 29 +++ 7 files changed, 458 insertions(+), 15 deletions(-) create mode 100644 install/featurebase.conf create mode 100644 install/featurebase.debian.service create mode 100644 install/featurebase.redhat.service create mode 100644 install/test_installation.Dockerfile create mode 100644 install/test_installation.sh diff --git a/Makefile b/Makefile index e2f659cf2..55d1332ba 100644 --- a/Makefile +++ b/Makefile @@ -104,10 +104,19 @@ build: # Create a single release build under the build directory release-build: $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/featurebase-$(VERSION_ID)/featurebase" - cp NOTICE README.md LICENSE build/featurebase$(VERSION_ID) + cp NOTICE install/featurebase.conf install/featurebase*.service build/featurebase-$(VERSION_ID) tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ @echo Created release build: build/featurebase-$(VERSION_ID).tar.gz +test-release-build: docker-build + mv build/featurebase-$(VERSION_ID).tar.gz install/ + cd install && docker build -t featurebase:test_installation \ + -f test_installation.Dockerfile \ + --build-arg release_tarball=featurebase-$(VERSION_ID).tar.gz . + mv install/featurebase-$(VERSION_ID).tar.gz build/ + docker run -it -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + featurebase:test_installation + # Error out if there are untracked changes in Git check-clean: ifndef SKIP_CHECK_CLEAN @@ -220,7 +229,7 @@ docker-build: vendor docker create --name featurebase-build featurebase:build mkdir -p build/featurebase-$(VERSION_ID) docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID) - cp NOTICE LICENSE ./build/featurebase-$(VERSION_ID) + cp NOTICE install/featurebase.conf install/featurebase*.service ./build/featurebase-$(VERSION_ID) docker rm featurebase-build tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ diff --git a/NOTICE b/NOTICE index 594272a27..cb62dfdca 100644 --- a/NOTICE +++ b/NOTICE @@ -1,23 +1,12 @@ Software license ================ -Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2017-2021 Molecula Corp. All rights reserved. Third-party software licenses ============================= -The file /pilosa/lru/lru.go contains a redistribution of lru +The file /lru/lru.go contains a redistribution of lru (github.com/golang/groupcache/lru); the license follows: Copyright 2013 Google Inc. diff --git a/install/featurebase.conf b/install/featurebase.conf new file mode 100644 index 000000000..73895ed6d --- /dev/null +++ b/install/featurebase.conf @@ -0,0 +1,373 @@ +# FEATUREBASE HOST CONFIGURATION +# +# Uncomment when/where appropriate + +# ============================================================================== +# Use advertise to specify the address advertised by the server to other nodes +# in the cluster and to clients via /status endpoint. Host defaults to IP +# address represented by bind parameter with network port. +# +# advertise = :10101 +# advertise-grpc = :20101 + + + +# "long-query-time" represents duration of time that will trigger log and stat +# message for queries longer than X time. Ex. "1m30s" 1 minute 30 seconds +# +# long-query-time = "10s" + + + +# Unique name for node in cluster. This is just a human-readable label for +# convenience and not used by any underlying logic. +# +# name = "featurebase1" + + + +# Host:Port where Featurebase server listens for HTTP requests. +# Default is localhost:10101 +# +# bind = "localhost:10101" + + + +# The address and port featurebase will listen to for all GRPC connections +# Ex. python-molecula, grafana for queries, etc. +# +# bind-grpc = "0.0.0.0:20101" + + + +# Directory to store Featurebase data files +data-dir = "/var/lib/molecula" + + +# ============================================================================== +# CORS (Cross-Origin Resource Sharing) Allowed Origins +# List of allowed origin URIs for CORS +# +# [handler] +# allowed-origins = ["https://myapp.com", "https://myapp.org"] + + + +# Path to the log file +log-path = "/var/log/molecula/featurebase.log" + + + +# Verbose - Enable verbose logging. Valid options are true or false. +# Set to true only when debugging as directed by Molecula engineers. +# +# verbose = true + + + +# Soft limit on max number of files featurebase will keep open simultaneously. +# When past this limit, featurebase will only keep files open for as long as is +# needed to write updates. +# +# max-file-count = 900000 + + + +# Maximum number of active memory maps featurebase will use for fragment files. +# Actual total usage may be slightly higher. +# Best practice is to set this to ~10% lower than your system's max map count. +# See sysctl vm.max_map_count in Linux. +# +# max-map-count = 900000 + + + +# Max Writes Per Request - Max number of mutating commands allowed per request. +# This includes Set, Clear, ClearRow, and Store +# +# max-writes-per-request = 5000 + + + +# The following option sets the maximum number of queries that are maintained +# for the /query-history endpoint. +# This parameter is per-node, and the result combines the history from all nodes. +# +# query-history-length = 100 + + + +# External database to connect to for `ExternalLookup` queries. +# lookup-db-dsn = "postgres://localhost:5432/db" + + + +# ============================================================================== +# For cluster stanza, "name" represents name for cluster. Must be same on all +# nodes in cluster. "replicas" represents number of hosts each piece of data +# should be stored on. Must be greater than or equal to 1 & less than or equal +# to number of nodes in cluster. +# [cluster] +# name = "cluster1" +# replicas = 1 + + + +# ============================================================================== +# [etcd] +# etcd is the tool Featurebase uses for node-to-node, intra-cluster +# communication. etcd is embedded in the featurebase cluster rather than +# running as a separate instance. +# It's important to configure this correctly for your network and nodes, and +# that it is consistent across all nodes. +# +# The easiest setup can be used when all nodes can reach all other nodes via a +# local subnet: +# listen-peer-address = advertise-peer-address +# = (what's in the initial-cluster-list) +# = the nodes ip address (which can be reached by every +# other node +# (localhost:10401 would not work for this, as each node can't reach that) +# +# If each node is separated by a proxy, or must be reached via url / dns, you +# will need to use a more complicated setup: +# listen-peer-address = the nodes local ip address +# (specific ip, localhost, or 0.0.0.0 for all +# local ip's) +# advertise-peer-address = the nodes ip address, reachable by all other nodes +# (This address should also be included in +# initital-cluster-list) +# in this case, you specify a different url/ip for listen-peer and +# advertise-peer. E.g. you specify 0.0.0.0 for listen, or (like in their case) +# you use a url for advertise. In each of these cases, you should set listen +# to the local ip, and you set advertise = to how each other node connects to +# this node, and you also use this same address in the initial cluster. +# The key here is that initial-cluster has to include the same node name and +# advertise-peer address as the node it's on (edited) + + + +# for additional assistance, and for help with config issues, +# see https://etcd.io/docs/v3.5/faq/ cluster-url - URL of existing cluster +# that a new node should join when adding nodes to cluster. +# +# cluster-url = "http://localhost:10401" + + + +# Address and port to bind to for client communication +# listen-client-address = "http://localhost:10401" + + + +# Address and port to bind to for peer communication +# listen-peer-address = "http://localhost:10301" + + + +# Comma-separated list of node=address pairs that makes up initial cluster when +# first started. In each pair, "node" value (left side of = ) should match +# name of node specified by "name" configuration parameter +# +# initial-cluster = "featurebase1=http://localhost:10301" + + + +# ============================================================================== +# Profile Block Rate - Block Rate is passed directly to Go's +# runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 +# per rate nanoseconds. A value of "1" samples every event, and 0 disables +# profiling. +# +# block-rate = 10000000 + +# Profile Mutex Fraction - Mutex Fraction is passed directly to Go's +# runtime.SetMutexProfileFraction. 1/ fraction of events will be sampled. +# +# mutex-fraction = 100 + + + +# ============================================================================== +# PostgreSQL Section +# [postgres] +# +# Endpoint Bind - Address to bind a PostgreSQL wire protocol endpoint. +# No PostgreSQL endpoint will be exposed unless a bind address is specified. +# Requires Molecula v3.0 or newer. +# +# bind = "localhost:55432" + + + +# The PostgreSQL endpoint has support for a connection limit. +# This is generally not necessary, so it is disabled by default. +# +# connection-limit = 10000 + + + +# PostgreSQL Max Startup Packet Size - By default, the postgres endpoint +# uses an 8 MiB limit on incoming PostgreSQL startup packets. This should +# typically be sufficient, but may be exceeded if a client sends an unusually +# large amount of configuration data. Oversized startup packets are typically +# caused by connecting with a different protocol, e.g. HTTP. +# +# max-startup-size = 10000000 + + + +# PostgreSQL Timeouts +# In order to detect stalled clients, the PostgreSQL endpoint has connection +# read and write timeouts. There is also a startup timeout, which is used for +# connection setup. The read timeout does not impact idle connections. Idle +# connections will only be closed by the server if TCP keepalive reports a +# break in the connection. TCP keepalives use the default configuration +# provided by the host. +# Caution: Due to a limitation of the PostgreSQL wire protocol, +# raising the write timeout may delay the shutdown of a featurebase node. +# +# startup-timeout = "20s" +# read-timeout = "20s" +# write-timeout - "20s" + + + +# Postgres Endpoint TLS - TLS configuration for the PostgreSQL endpoint is +# structured the same as the TLS configuration for Featurebase's other endpoints, +# but placed under [postgres.tls]. If TLS is configured on the postgres endpoint, +# Featurebase will reject unsecured connections. +# [postgres.tls] +# certificate = "/srv/pilosa/certs/server.crt" +# key = "/srv/pilosa/certs/server.key" +# ca-certificate = "/srv/pilosa/certs/ca.crt" +# enable-client-verification = true + + + +# ============================================================================== +# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is +# calculated periodically in the background and accessed by the UI/usage +# endpoint. Since this disk scan can take a long and unpredictable amount of +# time, its timing behavior is specified in a relative, rather than absolute +# sense. That is, the duty cycle sets the percentage of time that is spent +# recalculating this cache. This setting affects the results received from +# the "/ui/usage" http endpoint, as well as all data file and memory usage +# values and graphs on the webui "tables" page + +# Special considerations: +# * If disk usage can be calculated quickly (less than 5 seconds), fresh +# results will be calculated when accessed +# * When disk usage takes longer to calculate, there is a minimum of one +# hour wait between cache recalculations +# Setting this value to 0 will completely disable the calculation of disk usage +# +# usage-duty-cycle = 20 + + + +# ============================================================================== +# Use [metric] stanza to define attributes for monitoring. +# [metric] +# Specify which service to use for collecting metrics. Valid options are: +# "statsd", "expvar", "prometheus", "none" +# +# service = "prometheus" + + + +# Remote host to send statsd metrics to. +# host = "localhost:8125" + + + +# The interval to send statsd metrics. +# poll-interval = "10s" + + + +# Debugging flag to enable to send diagnostic information to Featurebase +# developers. +# +# diagnostics = false + + + +# ============================================================================== +# TLS Certificate Section - Path to TLC certificate used for service HTTPS. +# Suffix should contain .crt or .pem +# +# [tls] +# certificate = "/srv/pilosa/certs/server.crt" +# TLS Certificate Key - Path to TLS certificate key for HTTPS. Suffix should +# be .key +# +# key = "/srv/pilosa/certs/server.key" + + + +# ============================================================================== +# Tracing Section +# [tracing] +# +# Jaeger sampler type. Valid options are: "const, "probabilistic", "ratelimiting", +# or "remote". Set to 'off' to disable tracing completely. +# +# sampler-type = "remote" + + +# Jaeger sampler parameter (number) +# sampler-param = 0.001 + + + +# Tracing Agent Host:Port +# agent-host-port = "localhost:6831" + + + +# ============================================================================== +# Configuration for the RBF storage format. +# [rbf] +# Maximum size for each RBF database file. +# Allocates virtual memory but does not preallocate physical disk space. +# If you get into the range where you have 16000 shards on a single node +# (across all indexes), you will need to lower this in order to not run out of +# virtual address space. +# +# max-db-size = 4294967296 + + + +# Maximum size for each RBF WAL file. +# Allocates virtual memory but does not preallocate physical disk space. +# This is the same as max-db-size, but for the write-ahead log. If you set it +# smaller, set max-wal-checkpoint-size to 1/2 of this (we will likely +# condense these options in the future). +# +# max-wal-size = 4294967296 + + + +# Minimum WAL size before WAL pages can be copied to the main database file. +# min-wal-checkpoint-size = 1048576 + + + +# Maximum WAL size before transactions are halted to copy WAL pages to the +# main database file. +# +# max-wal-checkpoint-size = 2147483648 + + +# ============================================================================== +# [storage] +# Sync all changes to the file system. +# Should not be changed in production systems unless you know what you are +# doing - Should always be on unless testing or possibly while performing a +# bulk import and you are not worried about data loss +# +# fsync = true + + +# ============================================================================== diff --git a/install/featurebase.debian.service b/install/featurebase.debian.service new file mode 100644 index 000000000..162264106 --- /dev/null +++ b/install/featurebase.debian.service @@ -0,0 +1,13 @@ +[Unit] +Description="Service for FeatureBase" +After=network.target + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +WantedBy=multi-user.target diff --git a/install/featurebase.redhat.service b/install/featurebase.redhat.service new file mode 100644 index 000000000..cf507d8b0 --- /dev/null +++ b/install/featurebase.redhat.service @@ -0,0 +1,12 @@ +[Unit] +Description="Service for FeatureBase" + +[Service] +RestartSec=30 +Restart=on-failure +EnvironmentFile= +User=molecula +ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf + +[Install] +WantedBy=multi-user.target diff --git a/install/test_installation.Dockerfile b/install/test_installation.Dockerfile new file mode 100644 index 000000000..5d9a93253 --- /dev/null +++ b/install/test_installation.Dockerfile @@ -0,0 +1,18 @@ +FROM fedora:rawhide + +RUN yum -y install systemd procps + +ARG release_tarball + +COPY $release_tarball . + +COPY test_installation.sh test_installation.sh + +RUN mkdir install && \ + tar -xf $release_tarball -C install --strip-components 1 && \ + cd install && \ + cp featurebase /usr/local/bin/featurebase && \ + cp featurebase.redhat.service /etc/systemd/system/featurebase.service && \ + cp featurebase.conf /etc/featurebase.conf + +CMD ["/bin/bash", "test_installation.sh"] diff --git a/install/test_installation.sh b/install/test_installation.sh new file mode 100644 index 000000000..4a301ea48 --- /dev/null +++ b/install/test_installation.sh @@ -0,0 +1,29 @@ +#!/bin/bash +tests_failed=0 + +# test validity of featurebase.service +systemd-analyze verify /etc/systemd/system/featurebase.service +exit_code=$? +if [ $exit_code -ne 0 ]; then + echo 'featurebase.redhat.service invalid' + tests_failed=1 +fi + +# test validity of featurebase.conf +mkdir /var/log/molecula +featurebase_bin='/usr/local/bin/featurebase' +config_file='/etc/featurebase.conf' +if ! $featurebase_bin -c $config_file holder /dev/null 2>&1; then + echo 'featurebase.conf is invalid' + tests_failed=1 +fi + +# print success if both passed +if [ $tests_failed -eq 0 ]; then + echo 'featurebase.redhat.service is valid' + echo 'featurebase.conf is valid' +fi + +# tests_failed set to 0 if none of the tests failed +# otherwise set to non-zero +exit $tests_failed From 27b73d2b7d7baf8acd2ae241a1400c48a30c6b49 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Thu, 18 Nov 2021 14:27:19 -0800 Subject: [PATCH 159/222] corrects typo on line 229 --- qa/simulacraData/simulacra_data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 6e3dffc7e..d227f921e 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -226,7 +226,7 @@ func countryField() { func generateDate() string { // generates a random date in format YYYY-MM-DDT00:00 - // does not include leap days or populata the hour/minute time + // does not include leap days or populate the hour/minute time year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 month := rand.Intn(12) + 1 day := rand.Intn(30) + 1 From ce13bb78ede558ff170aaea8abcee0a24e0b7a9c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Thu, 18 Nov 2021 17:09:52 -0600 Subject: [PATCH 160/222] fixed typo --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f94cbcc0e..8294d2a3e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -233,7 +233,7 @@ deploy node for linux amd64: - PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - echo $PUBLIC_IP - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec3-user@$PUBLIC_IP:. + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$PUBLIC_IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: - job: build for linux amd64 From 8f85a9e88d7fac080244f6a0946b2db42f7a33cb Mon Sep 17 00:00:00 2001 From: reesporte Date: Thu, 18 Nov 2021 17:27:13 -0600 Subject: [PATCH 161/222] fix bug where timestamp val is wrong in Min/Max --- executor.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index 74e565aed..e6c719eb5 100644 --- a/executor.go +++ b/executor.go @@ -1157,6 +1157,7 @@ func (e *executor) executeMin(ctx context.Context, qcx *Qcx, index string, c *pq if other.Count == 0 { return ValCount{}, nil } + return other, nil } @@ -7575,8 +7576,11 @@ func (vc *ValCount) smaller(other ValCount) ValCount { extra += other.Count } return ValCount{ - Val: vc.Val, - Count: vc.Count + extra, + Val: vc.Val, + Count: vc.Count + extra, + DecimalVal: vc.DecimalVal, + FloatVal: vc.FloatVal, + TimestampVal: vc.TimestampVal, } } @@ -7626,9 +7630,13 @@ func (vc *ValCount) larger(other ValCount) ValCount { extra += other.Count } return ValCount{ - Val: vc.Val, - Count: vc.Count + extra, + Val: vc.Val, + Count: vc.Count + extra, + DecimalVal: vc.DecimalVal, + FloatVal: vc.FloatVal, + TimestampVal: vc.TimestampVal, } + } func (vc *ValCount) decimalLarger(other ValCount) ValCount { From b5e5df8b58ba17356837fc3a87a012788676c374 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Thu, 18 Nov 2021 16:10:49 -0800 Subject: [PATCH 162/222] add error checking --- qa/simulacraData/simulacra_data.go | 83 ++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 15 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index d227f921e..0ca7e2a16 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -61,14 +61,23 @@ func ageField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) for i := 0; i < totalRecords; i++ { if rand.Intn(100) <= 20 { - writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") - if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if err != nil { + log.Fatalf("failed to write to age.csv %s", err) } + + if rand.Intn(10) == 1 { + _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") + if err != nil { + log.Fatalf("failed to write to age.csv %s", err) + } + } + } } writer.Flush() @@ -81,12 +90,20 @@ func ipField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) for i := 0; i < totalRecords; i++ { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if err != nil { + log.Fatalf("failed to write to ip.csv %s", err) + } + if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") + if err != nil { + log.Fatalf("failed to write to ip.csv %s", err) + } } } @@ -100,12 +117,20 @@ func arbIdField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) for i := 0; i < totalRecords; i++ { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if err != nil { + log.Fatalf("failed to write to identifier.csv %s", err) + } + if rand.Intn(10) == 1 { - writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") + if err != nil { + log.Fatalf("failed to write to identifier.csv %s", err) + } } } @@ -119,6 +144,7 @@ func timeField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) r := rand.New(rand.NewSource(rand.Int63())) @@ -134,18 +160,27 @@ func timeField() { date := generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + log.Fatalf("failed to write to time.csv %s", err) + } if rand.Intn(5) == 1 { date = generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + log.Fatalf("failed to write to time.csv %s", err) + } } if rand.Intn(10) == 1 { value := distro.Uint64() urlString = "https://www.test.com/" + strconv.FormatUint(value, 10) date = generateDate() - writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") + if err != nil { + log.Fatalf("failed to write to time.csv %s", err) + } } } @@ -160,6 +195,7 @@ func optInField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) var optString string @@ -172,14 +208,24 @@ func optInField() { optString = "0" } - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to optin.csv %s", err) + } + if rand.Intn(10) == 1 { if optString == "0" { optString = "1" - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to optin.csv %s", err) + } } else { optString = "0" - writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to optin.csv %s", err) + } } } } @@ -194,6 +240,7 @@ func countryField() { if err != nil { log.Fatalf("failed to create file %s", err) } + writer := bufio.NewWriter(csvFile) //populate countries @@ -208,12 +255,18 @@ func countryField() { if rand.Intn(2) == 1 { value := distro.Uint64() country := countryList[value] - writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to country.csv %s", err) + } if rand.Intn(10) == 1 { value = distro.Uint64() country = countryList[value] - writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") + if err != nil { + log.Fatalf("failed to write to country.csv %s", err) + } } } From f6b3a81ac3694b87fe0b8f460c83f90576768e68 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 05:16:30 -0600 Subject: [PATCH 163/222] made duration minutes for query rate --- cmd/random-query/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 5c3bc4ee0..22f5209ad 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -55,7 +55,7 @@ type RandomQueryConfig struct { TimeTo time.Time // parsed time TimeRange int64 // hours between parsed times Index string - QPS int + QPM int SrcFile string Duration time.Duration Target vegeta.Target @@ -111,7 +111,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPS, "qps", 10, "number of currernt requests per sec to simulate, default 10") + fs.IntVar(&cfg.QPM, "qps", 10, "number of currernt requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -179,7 +179,7 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - rate := vegeta.Rate{Freq: cfg.QPS, Per: time.Second} + rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Second} duration := cfg.Duration targeter := vegeta.NewStaticTargeter(cfg.Target) attacker := vegeta.NewAttacker() From b3b11638a32d6b80cba4406e2c3d328170b6355c Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 9 Nov 2021 08:23:28 -0600 Subject: [PATCH 164/222] remove outdated files at top level we're no longer Apache 2.0 licensed, or open source, so LICENSE and CONTRIBUTING.MD are gone. We track the changelog elsewhere, so that can go, and I don't think anyone has looked at the NOTES file in 3 years. I modified the NOTICE not to refer to the Apache license any more. --- CHANGELOG.md | 715 ------------------------------------------------ CONTRIBUTING.md | 177 ------------ LICENSE | 202 -------------- NOTES | 26 -- 4 files changed, 1120 deletions(-) delete mode 100644 CHANGELOG.md delete mode 100644 CONTRIBUTING.md delete mode 100644 LICENSE delete mode 100644 NOTES diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 57de1f746..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,715 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/) -and this project adheres to [Semantic Versioning](http://semver.org/). - -## [1.4.0] - 2019-09-17 - -This version contains 99 contributions from 11 contributors. There are 94 files changed; 9,453 insertions; and 6,121 deletions. - -**Attention**: Pilosa 1.4.0 changes the way that integer fields are stored. The upgrade from old format to new is handled automatically, however you will not be able to downgrade to 1.3 should you wish to do so. We *always* recommend taking a backup of your Pilosa data directory before upgrading Pilosa, but doubly so with this release. - -### Added -- Update "Getting Started" documentation ([#2028](https://github.com/pilosa/pilosa/pull/2028)) -- Add ability to disable tracing and use nopTracer ([#2029](https://github.com/pilosa/pilosa/pull/2029)) -- Add test for no containers ([#2016](https://github.com/pilosa/pilosa/pull/2016)) -- Add naive implementations of Roaring and fuzz test ([#2023](https://github.com/pilosa/pilosa/pull/2023)) -- Add fuzzing code and readme.md to explain the fuzzer ([#2004](https://github.com/pilosa/pilosa/pull/2004)) -- Add MinRow and MaxRow calls ([#1983](https://github.com/pilosa/pilosa/pull/1983)) -- Add Prometheus stats backend ([#1992](https://github.com/pilosa/pilosa/pull/1992)) -- Add extra tracing spans and metadata ([#1939](https://github.com/pilosa/pilosa/pull/1939)) -- Add more Debugf() statements to the holder open process ([#1950](https://github.com/pilosa/pilosa/pull/1950)) -- Add ability to post schema using holder.applySchema ([#1956](https://github.com/pilosa/pilosa/pull/1956)) - -### Changed -- Update CircleCI build with Go 1.13 and run enterprise tests ([#2064](https://github.com/pilosa/pilosa/pull/2064)) -- Update Alpine to 3.9.4 in Dockerfile ([#2001](https://github.com/pilosa/pilosa/pull/2001)) -- Add Prometheus tests, refactor http stats as middleware, minor fixes ([#1994](https://github.com/pilosa/pilosa/pull/1994)) -- Add confirmation logic to catch false nodeLeave events ([#1993](https://github.com/pilosa/pilosa/pull/1993)) -- Improve TopN() errors ([#1978](https://github.com/pilosa/pilosa/pull/1978)) -- Make integer fields unbounded by using sign+magnitude representation ([#1902](https://github.com/pilosa/pilosa/pull/1902)) -- Simplify contributing instructions by removing weird upstream thing ([#1966](https://github.com/pilosa/pilosa/pull/1966)) - -### Fixed -- Default BSI base value to min, max, or 0 depending on the min/max range ([#2050](https://github.com/pilosa/pilosa/pull/2050)) -- Add worker pool for query processing ([#2034](https://github.com/pilosa/pilosa/pull/2034)) -- Move Range deprecation message to higher level ([#2033](https://github.com/pilosa/pilosa/pull/2033)) -- Use lock in view.deleteFragment while altering fragments ([#2026](https://github.com/pilosa/pilosa/pull/2026)) -- Fix malformed offset bug in readOffsets and readWithRuns ([#2021](https://github.com/pilosa/pilosa/pull/2021)) -- Fix various container iteration bugs in Roaring ([#2019](https://github.com/pilosa/pilosa/pull/2019)) -- Fix malformed bitmap handling ([#2017](https://github.com/pilosa/pilosa/pull/2017)) -- Fix fuzzer errors in roaring ([#2012](https://github.com/pilosa/pilosa/pull/2012)) -- Save all state files atomically to avoid corruption ([#2000](https://github.com/pilosa/pilosa/pull/2000)) -- Fix slice container updates ([#1997](https://github.com/pilosa/pilosa/pull/1997)) -- Fix out of bounds panic to show error ([#1975](https://github.com/pilosa/pilosa/pull/1975)) -- Fix error message returned by regex on field and index names ([#1973](https://github.com/pilosa/pilosa/pull/1973)) -- Fix filter calls in GroupBy not being translated ([#1970](https://github.com/pilosa/pilosa/pull/1970)) -- Fix TranslateFile behavior when reopened ([#1954](https://github.com/pilosa/pilosa/pull/1954)) -- Remove buggy shard validation code ([#1951](https://github.com/pilosa/pilosa/pull/1951)) -- Fix some lint warnings raised in VS-Code ([#1947](https://github.com/pilosa/pilosa/pull/1947)) - -### Performance -- Address some startup speed and performance issues ([#1988](https://github.com/pilosa/pilosa/pull/1988)) -- Add a worker pool for importRoaring jobs ([#2048](https://github.com/pilosa/pilosa/pull/2048)) -- Use UnionInPlace for computing time rows which involve multiple views ([#2041](https://github.com/pilosa/pilosa/pull/2041)) -- Improve ingest performance with snapshot queue and unmarshaling improvements ([#2024](https://github.com/pilosa/pilosa/pull/2024)) -- Improve row cache ([#1974](https://github.com/pilosa/pilosa/pull/1974)) - -### Removed -- Remove extraneous stat tags to improve prometheus performance ([#1996](https://github.com/pilosa/pilosa/pull/1996)) - -## [1.3.1] - 2019-05-01 - -This version contains 1 contribution from 1 contributor. There are 6 files changed; 10 insertions; and 95 deletions. - -### Fixed -- Remove shard validation to fix bug where some nodes weren't loading their fragments. #1951 ([#1964](https://github.com/pilosa/pilosa/pull/1964)) - -## [1.3.0] - 2019-04-16 - -This version contains 98 contributions from 10 contributors. There are 144 files changed; 12,635 insertions; and 4,341 deletions. - -### Added -- Add license headers and CI check ([#1940](https://github.com/pilosa/pilosa/pull/1940)) -- Add support to modify shard width at build time ([#1921](https://github.com/pilosa/pilosa/pull/1921)) -- Add 'bench' Makefile target and run fewer concurrency level benchmarks ([#1915](https://github.com/pilosa/pilosa/pull/1915)) -- Add server stats to /info endpoint ([#1859](https://github.com/pilosa/pilosa/pull/1859)) -- Implement config options for block profile rate and mutex fraction ([#1910](https://github.com/pilosa/pilosa/pull/1910)) -- Implement global open file counter using syswrap (to scale past system open file limits) ([#1906](https://github.com/pilosa/pilosa/pull/1906)) -- Implement global mmap counter with fallback (to scale past system mmap limits) ([#1903](https://github.com/pilosa/pilosa/pull/1903)) -- Add shard width to index info in schema (allows client to get shard width at run time) ([#1881](https://github.com/pilosa/pilosa/pull/1881)) -- Add shift operator ([#1761](https://github.com/pilosa/pilosa/pull/1761)) -- Support advertise address and listen on 0.0.0.0 ([#1832](https://github.com/pilosa/pilosa/pull/1832)) -- Added convenience function to efficiently calculate size of a roaring bitmap in bytes ([#1839](https://github.com/pilosa/pilosa/pull/1839)) -- Make sure more tests and benchmarks can have their temp dir set by flag ([#1831](https://github.com/pilosa/pilosa/pull/1831)) -- Add sliceascending/slicedescending striped benchmarks ([#1763](https://github.com/pilosa/pilosa/pull/1763)) -- Add setValue test and benchmarks ([#1820](https://github.com/pilosa/pilosa/pull/1820)) -- Add a test for groupby filter with RangeLTLT ([#1818](https://github.com/pilosa/pilosa/pull/1818)) -- Add tests for GroupBy with keys; removes unused Bit message from proto ([#1811](https://github.com/pilosa/pilosa/pull/1811)) - -### Fixed -- Update to latest memberlist fork with race fixes ([#1944](https://github.com/pilosa/pilosa/pull/1944)) -- Return original error instead of cause in handler ([#1943](https://github.com/pilosa/pilosa/pull/1943)) -- Validate (and panic) on duplicate PQL arguments ([#1938](https://github.com/pilosa/pilosa/pull/1938)) -- Add correct content type to query responses Fixes #1873 ([#1936](https://github.com/pilosa/pilosa/pull/1936)) -- Address race condition by getting cluster nodes with lock ([#1931](https://github.com/pilosa/pilosa/pull/1931)) -- Make sure to unmap containers before modifying ([#1876](https://github.com/pilosa/pilosa/pull/1876)) -- Avoid probable race when creating fragments ([#1863](https://github.com/pilosa/pilosa/pull/1863)) -- Improve help strings for metrics options ([#1887](https://github.com/pilosa/pilosa/pull/1887)) -- Ensure ClearRow() arguments get translated ([#1848](https://github.com/pilosa/pilosa/pull/1848)) -- Prevent omitting zero ids on columnattrs ([#1846](https://github.com/pilosa/pilosa/pull/1846)) -- Set cache size to 0 if cache type is none ([#1842](https://github.com/pilosa/pilosa/pull/1842)) -- Prevent deadlock in replication logic on reopening a store ([#1834](https://github.com/pilosa/pilosa/pull/1834)) -- Pass loggers around properly in gossip ([#1835](https://github.com/pilosa/pilosa/pull/1835)) -- Include read lock in cluster.Nodes() ([#1836](https://github.com/pilosa/pilosa/pull/1836)) -- Raise an error on Rows() query against a time field with noStandardView: true ([#1826](https://github.com/pilosa/pilosa/pull/1826)) -- Don't delete test fragment data (part of repo) ([#1827](https://github.com/pilosa/pilosa/pull/1827)) -- Fix bug on upper end of bsi range queries ([#1822](https://github.com/pilosa/pilosa/pull/1822)) -- Group by fixes ([#1802](https://github.com/pilosa/pilosa/pull/1802)) - -### Changed -- Switch to GolangCI lint ([#1924](https://github.com/pilosa/pilosa/pull/1924)) -- Return empty result set when query empty ([#1937](https://github.com/pilosa/pilosa/pull/1937)) -- Add Go 1.12 to CircleCI ([#1909](https://github.com/pilosa/pilosa/pull/1909)) -- Ignore fragment files from shards node doesn't own ([#1900](https://github.com/pilosa/pilosa/pull/1900)) -- Go module support. Use Modules instead of dep for dependencies ([#1616](https://github.com/pilosa/pilosa/pull/1616)) -- Merge Range() into Row() call. ([#1804](https://github.com/pilosa/pilosa/pull/1804)) -- Add from/to range arguments to Rows() call ([#1851](https://github.com/pilosa/pilosa/pull/1851)) -- Fixes Store call error messages, Rows doesn't need field argument ([#1830](https://github.com/pilosa/pilosa/pull/1830)) - -### Performance -- BTree performance improvements ([#1916](https://github.com/pilosa/pilosa/pull/1916)) -- Make Containers smaller, especially when they have small contents ([#1901](https://github.com/pilosa/pilosa/pull/1901)) -- Address UnionInPlace performance regressions ([#1897](https://github.com/pilosa/pilosa/pull/1897)) -- Small write path for import-roaring. Makes small imports faster ([#1892](https://github.com/pilosa/pilosa/pull/1892)) -- Small write path for imports ([#1871](https://github.com/pilosa/pilosa/pull/1871)) -- Remove copy for pilosa roaring files ([#1865](https://github.com/pilosa/pilosa/pull/1865)) -- Disable anti-entropy if not using replication [performance] ([#1814](https://github.com/pilosa/pilosa/pull/1814)) -- Group By—skip 0 counts as early as possible ([#1803](https://github.com/pilosa/pilosa/pull/1803)) - -## [1.2.0] - 2018-12-20 - -This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. - -### Added - -- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773)) -- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774)) -- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771)) -- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753)) -- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751)) -- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) -- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684)) -- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733)) -- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743)) -- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717)) -- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713)) -- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707)) -- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) -- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699)) -- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663)) -- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) -- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666)) -- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671)) -- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673)) -- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645)) -- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658)) -- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653)) -- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) -- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738)) -- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635)) -- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) -- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) -- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628)) - -### Changed - -- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780)) -- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) -- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748)) -- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740)) -- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725)) -- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724)) -- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677)) -- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) - -### Performance - -- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619)) -- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741)) - -### Fixed - -- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795)) -- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787)) -- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790)) -- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785)) -- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782)) -- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777)) -- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769)) -- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764)) -- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750)) -- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749)) -- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732)) -- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744)) -- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742)) -- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737)) -- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729)) -- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719)) -- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727)) -- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712)) -- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711)) -- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674)) -- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675)) -- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696)) -- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686)) -- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679)) -- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676)) -- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665)) -- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662)) -- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655)) -- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656)) -- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651)) -- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612)) -- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621)) -- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620)) -- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614)) - - -## [1.1.0] - 2018-08-21 - -This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions. - -### Added - -- Add CircleCI ([#1610](https://github.com/pilosa/pilosa/pull/1610)) -- Add key translation to exports ([#1608](https://github.com/pilosa/pilosa/pull/1608)) -- Support importing key values ([#1599](https://github.com/pilosa/pilosa/pull/1599), [#1601](https://github.com/pilosa/pilosa/pull/1601)) -- Treat coordinator as primary translate store ([#1582](https://github.com/pilosa/pilosa/pull/1582)) -- Add DEGRADED cluster state and handle gossip NodeLeave events correctly ([#1584](https://github.com/pilosa/pilosa/pull/1584)) -- Add linters to gometalinter and fix related issues ([#1544](https://github.com/pilosa/pilosa/pull/1544), [#1543](https://github.com/pilosa/pilosa/pull/1543), [#1540](https://github.com/pilosa/pilosa/pull/1540), [#1539](https://github.com/pilosa/pilosa/pull/1539), [#1537](https://github.com/pilosa/pilosa/pull/1537), [#1536](https://github.com/pilosa/pilosa/pull/1536), [#1535](https://github.com/pilosa/pilosa/pull/1535), [#1534](https://github.com/pilosa/pilosa/pull/1534), [#1530](https://github.com/pilosa/pilosa/pull/1530), [#1529](https://github.com/pilosa/pilosa/pull/1529), [#1528](https://github.com/pilosa/pilosa/pull/1528), [#1526](https://github.com/pilosa/pilosa/pull/1526), [#1527](https://github.com/pilosa/pilosa/pull/1527)) -- Add mutex field type ([#1524](https://github.com/pilosa/pilosa/pull/1524)) -- Fragment rows() and rowsForColumn() ([#1532](https://github.com/pilosa/pilosa/pull/1532)) - -### Fixed - -- Fix race on replicationClosing channel ([#1607](https://github.com/pilosa/pilosa/pull/1607)) -- Prevent anti-entropy and cluster resize from running simultaneously ([#1586](https://github.com/pilosa/pilosa/pull/1586)) -- Require a valid port that isn't greater than 65,535 ([#1603](https://github.com/pilosa/pilosa/pull/1603)) -- Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602)) -- Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558)) - -## [1.0.2] - 2018-08-01 - -This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions. - -### Fixed - -- Fix documentation ([#1503](https://github.com/pilosa/pilosa/pull/1503), [#1495](https://github.com/pilosa/pilosa/pull/1495), [#1551](https://github.com/pilosa/pilosa/pull/1551)) -- Fix places where empty IndexOptions were being used ([#1547](https://github.com/pilosa/pilosa/pull/1547)) -- Fix translator syncing bug in cluster environments ([#1552](https://github.com/pilosa/pilosa/pull/1552)) -- Fix race condition in translate_test ([#1541](https://github.com/pilosa/pilosa/pull/1541)) -- Add IndexOptions to IndexInfo json response ([#1542](https://github.com/pilosa/pilosa/pull/1542)) -- Add proper locking to cluster code to prevent races ([#1533](https://github.com/pilosa/pilosa/pull/1533)) -- Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502)) -- Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555)) - -## [1.0.1] - 2018-07-11 - -This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions. - -### Fixed - -- Use `dep ensure -vendor-only` for build repeatability ([#1491](https://github.com/pilosa/pilosa/pull/1491)) -- Make sure time range views are calculated correctly across months ([#1485](https://github.com/pilosa/pilosa/pull/1485)) -- Fix up error handling, add a configurable timeout to http handler closing ([#1486](https://github.com/pilosa/pilosa/pull/1486)) -- Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483)) -- Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493)) - -## [1.0.0] - 2018-07-09 - -This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions. - -### Added - -- ID-Key Translation ([#1337](https://github.com/pilosa/pilosa/pull/1337)) -- Add CORS support to handler ([#1327](https://github.com/pilosa/pilosa/pull/1327)) - -### Changed - -- HTTP handler updates ([#1408](https://github.com/pilosa/pilosa/pull/1408), [#1399](https://github.com/pilosa/pilosa/pull/1399), [#1441](https://github.com/pilosa/pilosa/pull/1441), [#1375](https://github.com/pilosa/pilosa/pull/1375), [#1433](https://github.com/pilosa/pilosa/pull/1433), [#1444](https://github.com/pilosa/pilosa/pull/1444), [#1388](https://github.com/pilosa/pilosa/pull/1388), [#1309](https://github.com/pilosa/pilosa/pull/1309), [#1302](https://github.com/pilosa/pilosa/pull/1302), [#1304](https://github.com/pilosa/pilosa/pull/1304), [#1465](https://github.com/pilosa/pilosa/pull/1465), [#1466](https://github.com/pilosa/pilosa/pull/1466)) -- Refactor/improve tests ([#1437](https://github.com/pilosa/pilosa/pull/1437), [#1434](https://github.com/pilosa/pilosa/pull/1434), [#1435](https://github.com/pilosa/pilosa/pull/1435), [#1425](https://github.com/pilosa/pilosa/pull/1425), [#1418](https://github.com/pilosa/pilosa/pull/1418), [#1419](https://github.com/pilosa/pilosa/pull/1419), [#1413](https://github.com/pilosa/pilosa/pull/1413), [#1394](https://github.com/pilosa/pilosa/pull/1394), [#1387](https://github.com/pilosa/pilosa/pull/1387), [#1386](https://github.com/pilosa/pilosa/pull/1386), [#1378](https://github.com/pilosa/pilosa/pull/1378), [#1364](https://github.com/pilosa/pilosa/pull/1364), [#1348](https://github.com/pilosa/pilosa/pull/1348), [#1340](https://github.com/pilosa/pilosa/pull/1340), [#1297](https://github.com/pilosa/pilosa/pull/1297)) -- Simplify inter-node communication ([#1428](https://github.com/pilosa/pilosa/pull/1428), [#1427](https://github.com/pilosa/pilosa/pull/1427), [#1412](https://github.com/pilosa/pilosa/pull/1412), [#1398](https://github.com/pilosa/pilosa/pull/1398), [#1391](https://github.com/pilosa/pilosa/pull/1391), [#1389](https://github.com/pilosa/pilosa/pull/1389)) -- Make gossip's interface to Pilosa the API struct ([#1452](https://github.com/pilosa/pilosa/pull/1452)) -- Rename slice to shard ([#1426](https://github.com/pilosa/pilosa/pull/1426)) -- Clearbit for time fields ([#1424](https://github.com/pilosa/pilosa/pull/1424)) -- Update docs ([#1390](https://github.com/pilosa/pilosa/pull/1390), [#1329](https://github.com/pilosa/pilosa/pull/1329), [#1305](https://github.com/pilosa/pilosa/pull/1305), [#1296](https://github.com/pilosa/pilosa/pull/1296), [#1461](https://github.com/pilosa/pilosa/pull/1461)) -- Simplify server setup ([#1417](https://github.com/pilosa/pilosa/pull/1417), [#1393](https://github.com/pilosa/pilosa/pull/1393),[#1451](https://github.com/pilosa/pilosa/pull/1451)) -- Refactor API ([#1407](https://github.com/pilosa/pilosa/pull/1407)) -- Rewrite PQL parser and add various improvements/simplifications ([#1382](https://github.com/pilosa/pilosa/pull/1382), [#1402](https://github.com/pilosa/pilosa/pull/1402), [#1354](https://github.com/pilosa/pilosa/pull/1354), [#1463](https://github.com/pilosa/pilosa/pull/1463)) -- Rename "frame" to "field" ([#1395](https://github.com/pilosa/pilosa/pull/1395), [#1362](https://github.com/pilosa/pilosa/pull/1362), [#1360](https://github.com/pilosa/pilosa/pull/1360), [#1358](https://github.com/pilosa/pilosa/pull/1358), [#1357](https://github.com/pilosa/pilosa/pull/1357), [#1355](https://github.com/pilosa/pilosa/pull/1355)) -- Optimize count ([#1365](https://github.com/pilosa/pilosa/pull/1365)) -- Simplify bitmap max function ([#1333](https://github.com/pilosa/pilosa/pull/1333)) -- Rename "bit" to "column" for clarity ([#1326](https://github.com/pilosa/pilosa/pull/1326)) -- Rename pilosa.Bitmap to Row ([#1311](https://github.com/pilosa/pilosa/pull/1311)) -- Invert encoding/decoding and remove internal references ([#1454](https://github.com/pilosa/pilosa/pull/1454)) - -### Removed - -- Rename (unexport) many items to reduce public API footprint prior to 1.0 release ([#1470](https://github.com/pilosa/pilosa/pull/1470), [#1458](https://github.com/pilosa/pilosa/pull/1458), [#1450](https://github.com/pilosa/pilosa/pull/1450), [#1449](https://github.com/pilosa/pilosa/pull/1449), [#1448](https://github.com/pilosa/pilosa/pull/1448), [#1447](https://github.com/pilosa/pilosa/pull/1447), [#1446](https://github.com/pilosa/pilosa/pull/1446), [#1438](https://github.com/pilosa/pilosa/pull/1438), [#1443](https://github.com/pilosa/pilosa/pull/1443), [#1440](https://github.com/pilosa/pilosa/pull/1440), [#1439](https://github.com/pilosa/pilosa/pull/1439), [#1409](https://github.com/pilosa/pilosa/pull/1409), [#1392](https://github.com/pilosa/pilosa/pull/1392), [#1374](https://github.com/pilosa/pilosa/pull/1374), [#1372](https://github.com/pilosa/pilosa/pull/1372), [#1369](https://github.com/pilosa/pilosa/pull/1369), [#1367](https://github.com/pilosa/pilosa/pull/1367), [#1366](https://github.com/pilosa/pilosa/pull/1366), [#1351](https://github.com/pilosa/pilosa/pull/1351), [#1420](https://github.com/pilosa/pilosa/pull/1420), [#1416](https://github.com/pilosa/pilosa/pull/1416), [#1397](https://github.com/pilosa/pilosa/pull/1397)) -- Remove dead code ([#1432](https://github.com/pilosa/pilosa/pull/1432), [#1457](https://github.com/pilosa/pilosa/pull/1457), [#1421](https://github.com/pilosa/pilosa/pull/1421), [#1411](https://github.com/pilosa/pilosa/pull/1411), [#1377](https://github.com/pilosa/pilosa/pull/1377), [#1393](https://github.com/pilosa/pilosa/pull/1393), [#1462](https://github.com/pilosa/pilosa/pull/1462)) -- Remove view argument from Field.SetBit and Field.ClearBit ([#1396](https://github.com/pilosa/pilosa/pull/1396)) -- Remove WebUI (now contained in a separate package) ([#1363](https://github.com/pilosa/pilosa/pull/1363)) -- Remove bench command ([#1347](https://github.com/pilosa/pilosa/pull/1347)) -- Remove "view" from API, handler, docs ([#1346](https://github.com/pilosa/pilosa/pull/1346)) -- Remove backup/restore stuff ([#1339](https://github.com/pilosa/pilosa/pull/1339), [#1341](https://github.com/pilosa/pilosa/pull/1341)) -- Remove inverse frame functionality ([#1335](https://github.com/pilosa/pilosa/pull/1335)) -- Remove rangeEnabled option ([#1332](https://github.com/pilosa/pilosa/pull/1332)) -- Remove index and field MarshalJSON ([#1468](https://github.com/pilosa/pilosa/pull/1468)) - -### Fixed - -- Fix a few data races ([#1423](https://github.com/pilosa/pilosa/pull/1423)) -- Fix for crash while removing containers ([#1401](https://github.com/pilosa/pilosa/pull/1401)) -- Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415)) -- Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350)) - -## [0.10.0] - 2018-05-15 - -This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions. - -### Added - -- Add B+Tree containers (Enterprise Edition) ([#1285](https://github.com/pilosa/pilosa/pull/1285)) -- Add /info endpoint ([#1236](https://github.com/pilosa/pilosa/pull/1236)) - -### Changed - -- Wrap errors ([#1271](https://github.com/pilosa/pilosa/pull/1271), [#1258](https://github.com/pilosa/pilosa/pull/1258), [#1274](https://github.com/pilosa/pilosa/pull/1274), [#1270](https://github.com/pilosa/pilosa/pull/1270), [#1273](https://github.com/pilosa/pilosa/pull/1273), [#1272](https://github.com/pilosa/pilosa/pull/1272), [#1260](https://github.com/pilosa/pilosa/pull/1260), [#1259](https://github.com/pilosa/pilosa/pull/1259), [#1256](https://github.com/pilosa/pilosa/pull/1256), [#1257](https://github.com/pilosa/pilosa/pull/1257), [#1261](https://github.com/pilosa/pilosa/pull/1261), [#1262](https://github.com/pilosa/pilosa/pull/1262), [#1263](https://github.com/pilosa/pilosa/pull/1263), [#1265](https://github.com/pilosa/pilosa/pull/1265)) - -### Removed - -- Remove unused code ([#1286](https://github.com/pilosa/pilosa/pull/1286)) -- Remove input definition, add install-stringer to Makefile ([#1284](https://github.com/pilosa/pilosa/pull/1284)) -- Remove /id and /hosts endpoints. Add local ID to /status ([#1238](https://github.com/pilosa/pilosa/pull/1238)) -- Remove API.URI ([#1255](https://github.com/pilosa/pilosa/pull/1255)) - -### Fixed - -- Assorted docs fixes ([#1281](https://github.com/pilosa/pilosa/pull/1281), [#1269](https://github.com/pilosa/pilosa/pull/1269)) -- Update PQL syntax in bench subcommand ([#1279](https://github.com/pilosa/pilosa/pull/1279)) -- Update help menu in WebUI ([#1278](https://github.com/pilosa/pilosa/pull/1278)) -- Fix dead lock ([#1268](https://github.com/pilosa/pilosa/pull/1268)) -- Make sure gossipMemberSet.Logger is set during server setup ([#1266](https://github.com/pilosa/pilosa/pull/1266)) -- Make sure ~ is expanded in NewServer; BroadcastReceiver uses temp path ([#1242](https://github.com/pilosa/pilosa/pull/1242)) -- Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234)) -- Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253)) - -## [0.9.0] - 2018-05-04 - -This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions. - -*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.* - -### Added - -- Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) -- Update docs to include cluster-resize config and instructions ([#1088](https://github.com/pilosa/pilosa/pull/1088)) -- Add support for lists of gossip seeds for redundancy ([#1133](https://github.com/pilosa/pilosa/pull/1133)) -- Add HTTP Handler validation ([#1140](https://github.com/pilosa/pilosa/pull/1140), [#1121](https://github.com/pilosa/pilosa/pull/1121)) -- Add validation around node-remove conditions ([#1138](https://github.com/pilosa/pilosa/pull/1138)) -- broadcast.SendSync field creation and deletion to all nodes ([#1132](https://github.com/pilosa/pilosa/pull/1132)) -- Spread recalculate caches to all nodes. Fixes #1069 ([#1109](https://github.com/pilosa/pilosa/pull/1109)) -- Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064)) -- Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) -- Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) -- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) -- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) -- Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) -- Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) -- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) -- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250)) - -### Changed - -- Put Statik behind an interface ([#1163](https://github.com/pilosa/pilosa/pull/1163)) -- Refactor diagnostics, inject gopsutil dependency ([#1166](https://github.com/pilosa/pilosa/pull/1166)) -- Use boolean instead of address to configure coordinator ([#1158](https://github.com/pilosa/pilosa/pull/1158)) -- Put GCNotify behind an interface ([#1148](https://github.com/pilosa/pilosa/pull/1148)) -- Replace custom assembly bit functions with standard go ([#797](https://github.com/pilosa/pilosa/pull/797)) -- Improve roaring tests ([#1115](https://github.com/pilosa/pilosa/pull/1115)) -- Change configuration cluster.type (string) to cluster.disabled (bool) ([#1099](https://github.com/pilosa/pilosa/pull/1099)) -- Use NodeID instead of URI for node identification ([#1077](https://github.com/pilosa/pilosa/pull/1077)) -- Change gossip config from DefaultLocalConfig to DefaultWANConfig ([#1032](https://github.com/pilosa/pilosa/pull/1032)) -- Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027)) -- Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001)) -- Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) -- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) -- Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) -- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) -- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) -- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) -- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) - -### Removed - -- Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171)) -- Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142)) -- Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130)) -- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226)) -- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222)) -- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219)) -- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212)) -- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209)) -- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207)) - -### Fixed - -- Handle the scheme correctly in config.Bind ([#1143](https://github.com/pilosa/pilosa/pull/1143)) -- Prevent excessive sendSync (createView) messages. ([#1139](https://github.com/pilosa/pilosa/pull/1139)) -- Fix a shift logic bug in bitmapZeroRange ([#1110](https://github.com/pilosa/pilosa/pull/1110)) -- Fix node id validation on set-coordinator ([#1102](https://github.com/pilosa/pilosa/pull/1102)) -- Avoid overflow bug in differenceRunArray ([#1105](https://github.com/pilosa/pilosa/pull/1105)) -- Fix bug in NewServerCluster where each host was its own coordinator ([#1101](https://github.com/pilosa/pilosa/pull/1101)) -- Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084)) -- Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979)) -- Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) -- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) -- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) -- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) - -### Deprecated - -- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) - -### Performance - -- Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017)) - -## [0.8.8] - 2018-02-19 - -This version contains 1 contribution from 2 contributors. There are 4 files changed; 1,153 insertions; and 618 deletions. - -### Fixed - -- Bug fixes and improved test coverage in roaring ([#1118](https://github.com/pilosa/pilosa/pull/1118)) - -## [0.8.7] - 2018-02-12 - -This version contains 1 contribution from 1 contributors. There are 2 files changed; 84 insertions; and 4 deletions. - -### Fixed - -- Fix a shift logic bug in bitmapZeroRange ([#1111](https://github.com/pilosa/pilosa/pull/1111)) - -## [0.8.6] - 2018-02-09 - -This version contains 2 contributions from 2 contributors. There are 3 files changed; 171 insertions; and 6 deletions. - -### Fixed - -- Fix overflow bug in differenceRunArray [#1106](https://github.com/pilosa/pilosa/pull/1106) -- Fix bug where count and bitmap queries could return different numbers [#1083](https://github.com/pilosa/pilosa/pull/1083) - -## [0.8.5] - 2018-01-18 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 1 insertion, and 0 deletions. - -### Fixed - -- Bind Docker container on all interfaces ([#1061](https://github.com/pilosa/pilosa/pull/1061)) - -## [0.8.4] - 2018-01-10 - -This version contains 4 contributions from 3 contributors. There are 17 files changed; 974 insertions; and 221 deletions. - -### Fixed - -- Group the write operations in syncBlock by MaxWritesPerRequest ([#1038](https://github.com/pilosa/pilosa/pull/1038)) -- Change gossip config from memberlist.DefaultLocalConfig to memberlist.DefaultWANConfig ([#1033](https://github.com/pilosa/pilosa/pull/1033)) - -### Performance - -- Change AttrBlock handler calls to support protobuf instead of json ([#1046](https://github.com/pilosa/pilosa/pull/1046)) -- Use RLock instead of Lock in a few places ([#1042](https://github.com/pilosa/pilosa/pull/1042)) - -## [0.8.3] - 2017-12-12 - -This version contains 1 contribution from 1 contributor. There are 2 files changed; 59 insertions; and 42 deletions. - -### Fixed - -- Protect against accessing pointers to memory which was unmapped ([#1000](https://github.com/pilosa/pilosa/pull/1000)) - -## [0.8.2] - 2017-12-05 - -This version contains 1 contribution from 1 contributor. There are 15 files changed; 127 insertions; and 98 deletions. - -### Fixed - -- Modify initialization of HTTP client so only one instance is created ([#994](https://github.com/pilosa/pilosa/pull/994)) - -## [0.8.1] - 2017-11-15 - -This version contains 2 contributions from 2 contributors. There are 4 files changed; 27 insertions; and 14 deletions. - -### Fixed - -- Fix CountOpenFiles() fatal crash ([#969](https://github.com/pilosa/pilosa/pull/969)) -- Fix version check when local is greater than pilosa.com ([#968](https://github.com/pilosa/pilosa/pull/968)) - -## [0.8.0] - 2017-11-15 - -This version contains 31 contributions from 8 contributors. There are 84 files changed; 3,732 insertions; and 1,428 deletions. - -### Added - -- Diagnostics ([#895](https://github.com/pilosa/pilosa/pull/895)) -- Add docker-build make target for repeatable Docker-based builds ([#933](https://github.com/pilosa/pilosa/pull/933)) -- Add documentation on importing field values; fixes #924 ([#938](https://github.com/pilosa/pilosa/pull/938)) -- Add flag documentation and tests, remove "plugins.path" ([#942](https://github.com/pilosa/pilosa/pull/942)) -- Add TLS support ([#867](https://github.com/pilosa/pilosa/pull/867)) -- Add TLS cluster how to ([#898](https://github.com/pilosa/pilosa/pull/898)) -- Add support for gossip encryption ([#889](https://github.com/pilosa/pilosa/pull/889)) -- Add Recalculate Caches endpoint ([#881](https://github.com/pilosa/pilosa/pull/881)) -- Add search-friendly documentation for BSI range query syntax ([#955](https://github.com/pilosa/pilosa/pull/955)) - -### Changed - -- Remove unneeded Gopkg.toml constraints and update all dependencies ([#943](https://github.com/pilosa/pilosa/pull/943)) -- Remove row and column labels in webUI ([#884](https://github.com/pilosa/pilosa/pull/884)) -- Internal Client refactoring ([#892](https://github.com/pilosa/pilosa/pull/892)) -- Remove column/row labels for input definition ([#945](https://github.com/pilosa/pilosa/pull/945)) -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Fixed - -- Skip permissions test when run as root. Fixes #940 ([#941](https://github.com/pilosa/pilosa/pull/941)) -- Address "connection reset" issues in client ([#934](https://github.com/pilosa/pilosa/pull/934)) -- Fix field value import: Use signed int and respect field minimum ([#919](https://github.com/pilosa/pilosa/pull/919)) -- Constrain BoltDB to version rather than specific revision ([#887](https://github.com/pilosa/pilosa/pull/887)) -- Fix bug in environment variable format ([#882](https://github.com/pilosa/pilosa/pull/882)) -- Fix overflow in differenceRunBitmap ([#949](https://github.com/pilosa/pilosa/pull/949)) - -### Performance - -- Use FieldNotNull to improve efficiency of BETWEEN queries ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.2] - 2017-11-15 - -This version contains 1 contribution from 1 contributor. There is 1 file changed; 16 insertions; and 1 deletion. - -### Changed - -- Bump HTTP client's MaxIdleConns and MaxIdleConnsPerHost ([#920](https://github.com/pilosa/pilosa/pull/920)) - -## [0.7.1] - 2017-10-09 - -This version contains 3 contributions from 3 contributors. There are 14 files changed; 221 insertions; and 52 deletions. - -### Changed - -- Update dependencies and Go version ([#878](https://github.com/pilosa/pilosa/pull/878)) - -### Performance - -- Leverage not-null field to make BETWEEN queries more efficient ([#874](https://github.com/pilosa/pilosa/pull/874)) - -## [0.7.0] - 2017-10-03 - -This version contains 59 contributions from 9 contributors. There are 61 files changed; 5207 insertions; and 1054 deletions. - -### Added - -- Add HTTP API for fields ([#811](https://github.com/pilosa/pilosa/pull/811), [#856](https://github.com/pilosa/pilosa/pull/856)) -- Add HTTP API for delete views ([#785](https://github.com/pilosa/pilosa/pull/785)) -- Modify import endpoint to handle BSI field values ([#840](https://github.com/pilosa/pilosa/pull/840)) -- Add field Range() support to Executor ([#791](https://github.com/pilosa/pilosa/pull/791)) -- Support PQL Range() queries for fields ([#755](https://github.com/pilosa/pilosa/pull/755)) -- Add Sum() field query ([#778](https://github.com/pilosa/pilosa/pull/778)) -- Add documentation for BSI ([#861](https://github.com/pilosa/pilosa/pull/861)) -- Add BETWEEN for Range queries ([#847](https://github.com/pilosa/pilosa/pull/847)) -- Add Xor support for PQL ([#789](https://github.com/pilosa/pilosa/pull/789)) -- Enable auto-creating the schema on imports ([#837](https://github.com/pilosa/pilosa/pull/837)) -- Update client library docs ([#831](https://github.com/pilosa/pilosa/pull/831)) -- Handle SIGTERM signal ([#830](https://github.com/pilosa/pilosa/pull/830)) -- Add cluster config example to docs ([#806](https://github.com/pilosa/pilosa/pull/806)) -- Add ability to exclude attributes and bits in Bitmap queries ([#783](https://github.com/pilosa/pilosa/pull/783)) - -### Fixed - -- Fix panic when iterating over an empty run container ([#860](https://github.com/pilosa/pilosa/pull/860)) -- Fix row id zero bug ([#814](https://github.com/pilosa/pilosa/pull/814)) -- Fix cache invalidation bug ([#795](https://github.com/pilosa/pilosa/pull/795)) -- Set container.n in differenceRunRun ([#794](https://github.com/pilosa/pilosa/pull/794)) -- Fix infinite loop in bitmap-to-array conversion ([#779](https://github.com/pilosa/pilosa/pull/779)) -- Fix CountRange bug ([#773](https://github.com/pilosa/pilosa/pull/773)) - -### Deprecated - -- Remove support for row/column labels ([#839](https://github.com/pilosa/pilosa/pull/839)) - -### Performance - -- Refactor differenceRunArray ([#859](https://github.com/pilosa/pilosa/pull/859)) -- Update fragment.FieldSum to use roaring IntersectionCount() ([#841](https://github.com/pilosa/pilosa/pull/841)) -- Add roaring optimizations ([#842](https://github.com/pilosa/pilosa/pull/842)) -- Convert lock to read lock ([#848](https://github.com/pilosa/pilosa/pull/848)) -- Reduce Lock calls in executor ([#846](https://github.com/pilosa/pilosa/pull/846)) -- Implement container.flipBitmap() to improve differenceRunBitmap() ([#849](https://github.com/pilosa/pilosa/pull/849)) -- Reuse container storage on UnmarshalBinary to improve memory utilization ([#820](https://github.com/pilosa/pilosa/pull/820)) -- Improve WriteTo performance ([#812](https://github.com/pilosa/pilosa/pull/812)) - -## [0.6.0] - 2017-08-11 - -This version contains 14 contributions from 5 contributors. There are 28 files changed; 4,936 insertions; and 692 deletions. - -### Added - -- Add Run-length Encoding ([#758](https://github.com/pilosa/pilosa/pull/758)) - -### Changed - -- Make gossip the default broadcast type ([#750](https://github.com/pilosa/pilosa/pull/750)) - -### Fixed - -- Fix CountRange ([#759](https://github.com/pilosa/pilosa/pull/759)) -- Fix `differenceArrayRun` logic ([#674](https://github.com/pilosa/pilosa/pull/674)) - -## [0.5.0] - 2017-08-02 - -This version contains 65 contributions from 8 contributors (including 1 volunteer contributor). There are 79 files changed; 7,972 insertions; and 2,800 deletions. - -### Added - -- Set open file limit during Pilosa startup ([#748](https://github.com/pilosa/pilosa/pull/748)) -- Add Input Definition ([#646](https://github.com/pilosa/pilosa/pull/646)) -- Add cache type: None ([#745](https://github.com/pilosa/pilosa/pull/745)) -- Add panic recovery in top level HTTP handler ([#741](https://github.com/pilosa/pilosa/pull/741)) -- Count open file handles as a StatsD metric ([#636](https://github.com/pilosa/pilosa/pull/636)) -- Add coverage tools to Makefile ([#635](https://github.com/pilosa/pilosa/pull/635)) -- Add Holder test coverage ([#629](https://github.com/pilosa/pilosa/pull/629)) -- Add runtime memory metrics ([#600](https://github.com/pilosa/pilosa/pull/600)) -- Add sorting flag to import command ([#606](https://github.com/pilosa/pilosa/pull/606)) -- Add PQL support for field values (WIP) ([#721](https://github.com/pilosa/pilosa/pull/721)) -- Set and retrieve field values (WIP) ([#702](https://github.com/pilosa/pilosa/pull/702)) -- Add BSI range-encoding schema support (WIP) ([#670](https://github.com/pilosa/pilosa/pull/670)) - -### Changed - -- Move InternalPort config option to top-level ([#747](https://github.com/pilosa/pilosa/pull/747)) -- Switch from glide to dep for dependency management ([#744](https://github.com/pilosa/pilosa/pull/744)) -- Remove QueryRequest.Quantum since it is no longer used ([#699](https://github.com/pilosa/pilosa/pull/699)) -- Refactor test utilities into importable package ([#675](https://github.com/pilosa/pilosa/pull/675)) - -### Fixed - -- Add mutex for attribute cache ([#729](https://github.com/pilosa/pilosa/pull/729)) -- Use log-path flag to specify log file ([#678](https://github.com/pilosa/pilosa/pull/678)) - -## [0.4.0] - 2017-06-08 - -This version contains 53 contributions from 13 contributors (including 4 volunteer contributors). There are 96 files changed; 6373 insertions; and 770 deletions. - -*Note that data files created in Pilosa < 0.4.0 are not compatible with Pilosa 0.4.0 as a result of [#520](https://github.com/pilosa/pilosa/pull/520).* - -### Added -- Support metric reporting through StatsD protocol ([#468](https://github.com/pilosa/pilosa/pull/468), [#568](https://github.com/pilosa/pilosa/pull/568), [#580](https://github.com/pilosa/pilosa/pull/580)) -- Improve test coverage for ctl package ([#586](https://github.com/pilosa/pilosa/pull/586)) -- Add support for bit flip (negate) in roaring ([#592](https://github.com/pilosa/pilosa/pull/592)) -- Add xor support to roaring ([#571](https://github.com/pilosa/pilosa/pull/571)) -- Improve WebUI autocomplete ([#560](https://github.com/pilosa/pilosa/pull/560)) -- Add syntax hints tooltip to WebUI ([#537](https://github.com/pilosa/pilosa/pull/537)) -- Implement 'config' CLI command ([#541](https://github.com/pilosa/pilosa/pull/541)) -- Move docs into repo ([#563](https://github.com/pilosa/pilosa/pull/563)) -- Add inverse TopN() support ([#551](https://github.com/pilosa/pilosa/pull/551)) -- Add various Makefile updates ([#540](https://github.com/pilosa/pilosa/pull/540)) -- Provide details on Glide checksum mismatch ([#546](https://github.com/pilosa/pilosa/pull/546)) -- Add Docker multi-stage build ([#535](https://github.com/pilosa/pilosa/pull/535)) -- Support inverse Range() queries ([#533](https://github.com/pilosa/pilosa/pull/533)) -- Support colon commands in WebUI ([#529](https://github.com/pilosa/pilosa/pull/529), [#510](https://github.com/pilosa/pilosa/pull/510)) - -### Changed -- Increase default partition count from 16 to 256 (BREAKING CHANGE) ([#520](https://github.com/pilosa/pilosa/pull/520)) -- Validate unknown query params ([#578](https://github.com/pilosa/pilosa/pull/578)) -- Validate configuration file ([#573](https://github.com/pilosa/pilosa/pull/573)) -- Change default cache type to ranked ([#524](https://github.com/pilosa/pilosa/pull/524)) -- Add max-writes-per-requests limit ([#525](https://github.com/pilosa/pilosa/pull/525)) - -### Fixed -- Add "make test" to PHONY section of Makefile ([#605](https://github.com/pilosa/pilosa/pull/605)) -- Fix failing tests when IPv6 is disabled ([#594](https://github.com/pilosa/pilosa/pull/594)) -- Add minor docs fix, indent in JSON ([#599](https://github.com/pilosa/pilosa/pull/599)) -- Fix BroadcastHandler handle missing index error ([#597](https://github.com/pilosa/pilosa/pull/597)) -- Add WebUI fixes ([#589](https://github.com/pilosa/pilosa/pull/589)) -- Fix support for 32-bit Linux ([#549](https://github.com/pilosa/pilosa/pull/549), [#565](https://github.com/pilosa/pilosa/pull/565)) -- Fix 3 separate bugs in bitmapCountRange ([#559](https://github.com/pilosa/pilosa/pull/559)) -- Add client support for MaxInverseSliceByIndex ([#555](https://github.com/pilosa/pilosa/pull/555)) -- Fix bug in `handleGetSliceMax` ([#554](https://github.com/pilosa/pilosa/pull/554)) -- Default to `standard` view in export command ([#548](https://github.com/pilosa/pilosa/pull/548)) -- Fix vet issues with the assembly code in Roaring ([#528](https://github.com/pilosa/pilosa/pull/528)) -- Prevent row labels that match the column label ([#503](https://github.com/pilosa/pilosa/pull/503)) -- Fix roaring test: TestBitmap_Quick_Array1 ([#507](https://github.com/pilosa/pilosa/pull/507)) -- Don't try to create inverse views on Import() when inverseEnabled is false ([#462](https://github.com/pilosa/pilosa/pull/462)) - -### Performance -- Set n based on array length instead of incrementing repeatedly ([#590](https://github.com/pilosa/pilosa/pull/590)) -- Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577)) -- Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562)) - -[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD -[0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4 -[0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5 -[0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 -[0.7.0]: https://github.com/pilosa/pilosa/compare/v0.6...v0.7 -[0.8.0]: https://github.com/pilosa/pilosa/compare/v0.7...v0.8 -[0.9.0]: https://github.com/pilosa/pilosa/compare/v0.8...v0.9 -[0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10 -[1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0 -[1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1 -[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 8e4b45e4e..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,177 +0,0 @@ -# Contributing to Pilosa - -The workflow components of these instructions apply to all Pilosa repositories. - -## Reporting a bug - -If you have discovered a bug and don't see it in the [github issue tracker][5], [open a new issue][1]. - -## Submitting a feature request - -Feature requests are managed in Github issues, organized with [Zenhub](https://www.zenhub.com/), which is publicly available as a browser extension. New features typically go through a [Proposal Process][4] -which starts by [opening a new issue][1] that describes the new feature proposal. - -## Making code contributions - -Before you start working on new features, you should [open a new issue][1] to let others know what -you're doing, otherwise you run the risk of duplicating effort. This also -gives others an opportunity to provide input for your feature. - -If you want to help but you aren't sure where to start, check out our [github label for low-effort issues][6]. - - -### Development Environment - -- Ensure you have a recent version of [Go](https://golang.org/doc/install) installed. Pilosa generally supports the current and previous minor versions; check our [CircleCI config file](../master/.circleci/config.yml) for the most up-to-date information. - -- Make sure `$GOPATH` environment variable points to your Go working directory and `$PATH` incudes `$GOPATH/bin`, as described [here](https://golang.org/doc/code.html#GOPATH). - -- Fork the [Pilosa repository][2] to your own account. - -- It will be easier to follow these instructions if you: - - ```sh - export GH_USERNAME= - ``` - -- Create a directory (note that we use `github.com/pilosa`, NOT `github.com/USER`) and clone Pilosa: - - ```sh - mkdir -p ${GOPATH}/src/github.com/pilosa && cd $_ - git clone https://github.com/pilosa/pilosa.git - ``` - -- `cd` to your pilosa directory: - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - ``` - -- Install Pilosa command line tools: - - ```sh - make install - ``` - - Running `pilosa` should now run a Pilosa instance. - -- The official Pilosa repository is your "origin" remote in git. Add your fork as your github username - - ```sh - cd ${GOPATH}/src/github.com/pilosa/pilosa - git remote add ${GH_USERNAME} git@github.com:${GH_USERNAME}/pilosa.git - ``` - -### Makefile - -Pilosa includes a Makefile that automates several tasks: - -- Install Pilosa: - - ```sh - make install - ``` - -- Install build dependencies: - - ```sh - make install-build-deps - ``` - -- Create the vendor directory: - - ```sh - make vendor - ``` - -- Run the test suite: - - ```sh - make test - ``` - -- View the coverage report: - - ```sh - make cover-viz - ``` - -- Clear the `vendor/` and `build/` directories: - - ```sh - make clean - ``` - -- Create release tarballs: - - ```sh - make release - ``` - -- Regenerate protocol buffer files in `internal/`: - - ```sh - make generate-protoc - ``` - -- Create tagged Docker image: - - ```sh - make docker - ``` - -- Run tests inside Docker container: - - ```sh - make docker-test - ``` - -Additional commands are available in the `Makefile`. - -### Submitting code changes - -- Before starting to work on a task, sync your branch with the upstream: - - ```sh - git checkout master - git pull - ``` - -- Create a local feature branch: - - ```sh - git checkout -b something-amazing - ``` - -- Commit your changes locally using `git add` and `git commit`. Please use [appropriate commit messages](https://chris.beams.io/posts/git-commit/). - -- Make sure that you've written tests for your new feature, and then run the tests: - - ```sh - make test - ``` - -- Verify that your pull request is applied to the latest version of code on github: - - ```sh - git checkout master - git pull - git checkout something-amazing - git rebase master - ``` - -- Push to your fork: - - ```sh - git push -u $GH_USERNAME something-amazing:something-amazing - ``` - -- Submit a [pull request][3] - - -[1]: https://github.com/pilosa/pilosa/issues/new -[2]: https://github.com/pilosa/pilosa -[3]: https://github.com/pilosa/pilosa/compare/ -[4]: https://github.com/pilosa/general/blob/master/proposal.md -[5]: https://github.com/pilosa/pilosa/issues -[6]: https://github.com/pilosa/pilosa/issues?q=is%3Aopen+is%3Aissue+label%3Anewcomer diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/NOTES b/NOTES deleted file mode 100644 index 8da55fe11..000000000 --- a/NOTES +++ /dev/null @@ -1,26 +0,0 @@ - - Index Column - ┌───────────▼────────────────────────────┐ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - Row──▶0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │0000000000000000000000000000000000000000│ - │────────────────────────────────────────┤ - F ▶│0000000000000000000000000000000000000000│ - i ││0000000000000000000000000000000000000000│ - e ││0000000000000000000000000000000000000000│ - l ││0000000000000000000000000000000000000000│ - d ▶│0000000000000000000000000000000000000000│ - └────────────────────────────────────────┘ - ▲───────────▲ - Shard - - -Fragment=intersection of field & shard From 0adfa751883eaaa959a069a65ce95353bdf8a684 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 19 Nov 2021 09:23:01 -0600 Subject: [PATCH 165/222] add unit test we can avoid regressions with a simple unit test that checks that timestamp ValCounts have the appropriate values in comparisons --- executor.go | 1 - executor_internal_test.go | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index e6c719eb5..b361d9672 100644 --- a/executor.go +++ b/executor.go @@ -1157,7 +1157,6 @@ func (e *executor) executeMin(ctx context.Context, qcx *Qcx, index string, c *pq if other.Count == 0 { return ValCount{}, nil } - return other, nil } diff --git a/executor_internal_test.go b/executor_internal_test.go index ce860c46c..8337bb296 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -21,6 +21,7 @@ import ( "reflect" "strconv" "testing" + "time" "github.com/molecula/featurebase/v2/pql" "github.com/molecula/featurebase/v2/testhook" @@ -301,6 +302,20 @@ func TestValCountComparisons(t *testing.T) { expLarger: ValCount{FloatVal: 10.7, Count: 3}, expSmaller: ValCount{FloatVal: 10.7, Count: 3}, }, + { + name: "timestampEquality", + vc: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1}, + other: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1}, + expLarger: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2}, + expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 2}, + }, + { + name: "timestamp", + vc: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1}, + other: ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1}, + expLarger: ValCount{Val: 1587399600, TimestampVal: time.Unix(0, 1587399600*int64(time.Second)), Count: 1}, + expSmaller: ValCount{Val: -17782800, TimestampVal: time.Unix(0, -17782800*int64(time.Second)), Count: 1}, + }, } for i, test := range tests { From ff39b7614397745e1b8d73001bdad587afcfd068 Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 19 Nov 2021 09:32:41 -0600 Subject: [PATCH 166/222] remove extra line smh my head --- executor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/executor.go b/executor.go index b361d9672..009f55c7b 100644 --- a/executor.go +++ b/executor.go @@ -7635,7 +7635,6 @@ func (vc *ValCount) larger(other ValCount) ValCount { FloatVal: vc.FloatVal, TimestampVal: vc.TimestampVal, } - } func (vc *ValCount) decimalLarger(other ValCount) ValCount { From c72406819ee3eb46f056cf12c70429c42e2b675a Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 09:40:39 -0600 Subject: [PATCH 167/222] added seed param --- cmd/random-query/main.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 22f5209ad..7b01575ee 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -56,6 +56,7 @@ type RandomQueryConfig struct { TimeRange int64 // hours between parsed times Index string QPM int + Seed int SrcFile string Duration time.Duration Target vegeta.Target @@ -109,9 +110,10 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") + fs.IntVar(&cfg.Seed, "seed", 0, "RNG seed") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPM, "qps", 10, "number of currernt requests per minute to simulate, default 10") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of currernt requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -179,7 +181,7 @@ func (cfg *RandomQueryConfig) Run() (err error) { if err != nil { return err } - rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Second} + rate := vegeta.Rate{Freq: cfg.QPM, Per: time.Minute} duration := cfg.Duration targeter := vegeta.NewStaticTargeter(cfg.Target) attacker := vegeta.NewAttacker() @@ -348,8 +350,7 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") } - seed := int64(42) - cfg.Rnd = rand.New(rand.NewSource(seed)) + cfg.Rnd = rand.New(rand.NewSource(int64(cfg.Seed))) return cfg.buildPayload() } From 10894352181a27addae7c0c87d2adb714581cbcc Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 09:48:45 -0600 Subject: [PATCH 168/222] add check for index presence --- cmd/random-query/main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 7b01575ee..5fbe72290 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -113,7 +113,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.Seed, "seed", 0, "RNG seed") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPM, "qpm", 10, "number of currernt requests per minute to simulate, default 10") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -317,9 +317,10 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { return err } foundIntField := false + any := false for i, ii := range cfg.Info { if ii.Name == cfg.Index { - + any = true _ = i for k, fld := range ii.Fields { _ = k @@ -346,6 +347,9 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) { } } } + if !any { + return errors.New(fmt.Sprintf("index %v not found", cfg.Index)) + } cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} if foundIntField { cfg.BitmapFunc = append(cfg.BitmapFunc, "Distinct") From f7b4f621a15f3df150b3665b99787bddc20587b0 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 19 Nov 2021 10:38:06 -0600 Subject: [PATCH 169/222] remove references to LICENSE and checks for it in source files --- .circleci/config.yml | 11 --- .cloudbuild/Dockerfile | 1 - .cloudbuild/LICENSE | 202 ---------------------------------------- .cloudbuild/NOTICE | 13 +-- .gitlab/Dockerfile | 1 - Dockerfile | 1 - Dockerfile-clustertests | 1 - Dockerfile.pilosa | 1 - Makefile | 10 -- roaring/btree_test.go | 2 +- 10 files changed, 2 insertions(+), 241 deletions(-) delete mode 100644 .cloudbuild/LICENSE diff --git a/.circleci/config.yml b/.circleci/config.yml index 65534faa6..498c56ea3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,13 +63,6 @@ jobs: - checkout-plus - run: go mod download - save-mod-cache - check-license-headers: - executor: - name: golang - steps: - - checkout-plus - - skip-if-root-unchanged - - run: make check-license-headers linter: executor: name: golang @@ -239,10 +232,6 @@ workflows: context: molecula requires: - setup - - check-license-headers: - context: molecula - requires: - - setup - go-mod-tidy: context: molecula requires: diff --git a/.cloudbuild/Dockerfile b/.cloudbuild/Dockerfile index 407e5b994..fac3bf2fd 100644 --- a/.cloudbuild/Dockerfile +++ b/.cloudbuild/Dockerfile @@ -5,7 +5,6 @@ LABEL org.opencontainers.image.authors="dev@molecula.com" RUN apk add --no-cache curl jq -COPY LICENSE /LICENSE COPY NOTICE /NOTICE ARG SHORT_SHA diff --git a/.cloudbuild/LICENSE b/.cloudbuild/LICENSE deleted file mode 100644 index d64569567..000000000 --- a/.cloudbuild/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/.cloudbuild/NOTICE b/.cloudbuild/NOTICE index 594272a27..5d88ea9c0 100644 --- a/.cloudbuild/NOTICE +++ b/.cloudbuild/NOTICE @@ -1,18 +1,7 @@ Software license ================ -Copyright (C) 2017-2018 Pilosa Corp. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"). -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2017-2021 Molecula Corp. All rights reserved. Third-party software licenses ============================= diff --git a/.gitlab/Dockerfile b/.gitlab/Dockerfile index b22392ce1..b041f58ce 100644 --- a/.gitlab/Dockerfile +++ b/.gitlab/Dockerfile @@ -7,7 +7,6 @@ WORKDIR /featurebase RUN apk add --no-cache curl jq -COPY LICENSE . COPY NOTICE . COPY featurebase_linux_amd64 . diff --git a/Dockerfile b/Dockerfile index 115855034..25ea9c3f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,6 @@ RUN apk add --no-cache curl jq COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 8c60f11bf..4a3a0c196 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -20,7 +20,6 @@ RUN apt install -y docker.io RUN cp /go/bin/featurebase /featurebase -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Dockerfile.pilosa b/Dockerfile.pilosa index a24c70ec7..6ee2e0bbf 100644 --- a/Dockerfile.pilosa +++ b/Dockerfile.pilosa @@ -24,7 +24,6 @@ RUN apk add --no-cache curl jq COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY LICENSE /LICENSE COPY NOTICE /NOTICE EXPOSE 10101 diff --git a/Makefile b/Makefile index 55d1332ba..f72517684 100644 --- a/Makefile +++ b/Makefile @@ -18,10 +18,6 @@ GO_VERSION=1.16.10 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia -define LICENSE_HASH_CODE - head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/' -e 's/Pilosa Corp\./Molecula Corp./' | shasum | cut -f 1 -d " " -endef -LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go)) UNAME := $(shell uname -s) ifeq ($(UNAME), Darwin) IS_MACOS:=1 @@ -313,12 +309,6 @@ gometalinter: require-gometalinter vendor --exclude "^pql/pql.peg.go" \ ./... -# Verify that all Go files have license header -check-license-headers: SHELL:=/bin/bash -check-license-headers: - @! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\ - do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.' - ###################### # Build dependencies # ###################### diff --git a/roaring/btree_test.go b/roaring/btree_test.go index 7a952a937..ee51aeb0e 100644 --- a/roaring/btree_test.go +++ b/roaring/btree_test.go @@ -1,6 +1,6 @@ // Copyright 2014 The b Authors. All rights reserved. // Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. +// license that can be found in the NOTICE file. package roaring From b12f00db590dfe05549483219a0d087016d7ff43 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 10:57:36 -0600 Subject: [PATCH 170/222] changed default seed to current time --- cmd/random-query/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 5fbe72290..475ba9e60 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -110,7 +110,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") - fs.IntVar(&cfg.Seed, "seed", 0, "RNG seed") + fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to currentime") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") @@ -440,6 +440,7 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } + vprint.VV("Add: %s %v %v %v", field, min, max, scale) /* if min.Scale != scale || max.Scale != scale { PanicOn(fmt.Sprintf("scale error; %v:%v min scale %d, max scale %d, field scale %d, assumed they'd be equal", From 25e5c6f5fbc1b9d4729cf5dcd6c98812e2366c65 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 19 Nov 2021 11:22:15 -0600 Subject: [PATCH 171/222] remove references to CONTRIBUTING and CHANGELOG --- .github/PULL_REQUEST_TEMPLATE.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2c1f4979d..f579dc42f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,13 +6,10 @@ Fixes # ## Pull request checklist -- [ ] I have read the [contributing guide](https://github.com/molecula/featurebase/blob/master/CONTRIBUTING.md). -- [ ] I have agreed to the [Contributor License Agreement](https://cla-assistant.io/pilosa/pilosa). - [ ] I have updated the [documentation](https://github.com/molecula/docs). - [ ] I have resolved any merge conflicts. - [ ] I have included tests that cover my changes. - [ ] All new and existing tests pass. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. - [ ] Add appropriate changelog label to PR (if applicable). ## Code review checklist @@ -24,5 +21,4 @@ This is the checklist that the reviewer will follow while reviewing your pull re - [ ] Check that tests have been written and that they cover the new functionality. - [ ] Run tests and ensure they pass. - [ ] Build and run the code, performing any applicable integration testing. -- [ ] Make sure PR title conforms to convention in CHANGELOG.md. - [ ] Make sure PR is tagged with appropriate changelog label. From 989f9c3f48c480f58be9251c884131c5bdcae8c9 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 11:29:17 -0600 Subject: [PATCH 172/222] applied suggestions --- cmd/random-query/main.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 475ba9e60..0fcd71ce9 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -42,18 +42,16 @@ import ( // RandomQueryConfig type RandomQueryConfig struct { - - // user facing flags - HostPort string // -hostport - TreeDepth int // -d - QueryCount int // -n - Verbose bool // -v + HostPort string + TreeDepth int + QueryCount int + Verbose bool NumRuns int - TimeFromArg string // --time.from - TimeToArg string // --time.to - TimeFrom time.Time // parsed time - TimeTo time.Time // parsed time - TimeRange int64 // hours between parsed times + TimeFromArg string + TimeToArg string + TimeFrom time.Time + TimeTo time.Time + TimeRange int64 Index string QPM int Seed int @@ -110,10 +108,10 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.TreeDepth, "max-nesting-depth", 1, "depth of random queries to generate.") fs.IntVar(&cfg.QueryCount, "queries-per-request", 1, "number of random queries to generate") fs.IntVar(&cfg.NumRuns, "number-reports", 1, "number of reports generate ") - fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to currentime") + fs.IntVar(&cfg.Seed, "seed", int(time.Now().Unix()), "RNG seed, defaults to current time") fs.DurationVar(&cfg.Duration, "metrics-period", 10*time.Second, "size of time window on metrics reporting, default 10s") fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") - fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") + fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") @@ -142,6 +140,9 @@ func (c *RandomQueryConfig) ValidateConfig() error { return fmt.Errorf("time.to (%s) should be at least one hour after time.from (%s)", c.TimeToArg, c.TimeFromArg) } + if c.QPM <= 0 { + return fmt.Errorf("-qpm must be positive") + } return nil } From dc9c2a842aac63bf409e8c3d0d7f3c6f9ce734a8 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Fri, 19 Nov 2021 09:38:12 -0800 Subject: [PATCH 173/222] make error handling better --- qa/simulacraData/simulacra_data.go | 163 ++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 48 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 0ca7e2a16..f5741201a 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -21,7 +21,6 @@ package main import ( "bufio" "fmt" - "log" "math/rand" "os" "strconv" @@ -48,18 +47,40 @@ var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "AL const totalRecords int = 200000000 func main() { - ageField() - ipField() - arbIdField() - optInField() - countryField() - timeField() + err0 := ageField() + if err0 != nil { + fmt.Printf("unable to generate age field: %v", err0) + } + + err1 := ipField() + if err1 != nil { + fmt.Printf("unable to generate IPfield: %v", err1) + } + + err2 := arbIdField() + if err2 != nil { + fmt.Printf("unable to generate indentifier field: %v", err2) + } + + err3 := optInField() + if err3 != nil { + fmt.Printf("unable to generate opt in field: %v", err3) + } + err4 := countryField() + if err4 != nil { + fmt.Printf("unable to generate country field: %v", err4) + } + + err5 := timeField() + if err5 != nil { + fmt.Printf("unable to generate time field: %v", err5) + } } -func ageField() { +func ageField() error { csvFile, err := os.Create("age.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -68,27 +89,34 @@ func ageField() { if rand.Intn(100) <= 20 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { - log.Fatalf("failed to write to age.csv %s", err) + return err } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { - log.Fatalf("failed to write to age.csv %s", err) + return err } } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func ipField() { +func ipField() error { csvFile, err := os.Create("ip.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -96,26 +124,34 @@ func ipField() { for i := 0; i < totalRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { - log.Fatalf("failed to write to ip.csv %s", err) + return err } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { - log.Fatalf("failed to write to ip.csv %s", err) + return err } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func arbIdField() { +func arbIdField() error { csvFile, err := os.Create("identifier.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -123,26 +159,34 @@ func arbIdField() { for i := 0; i < totalRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { - log.Fatalf("failed to write to identifier.csv %s", err) + return err } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { - log.Fatalf("failed to write to identifier.csv %s", err) + return err } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func timeField() { +func timeField() error { csvFile, err := os.Create("time.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -162,14 +206,14 @@ func timeField() { _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - log.Fatalf("failed to write to time.csv %s", err) + return err } if rand.Intn(5) == 1 { date = generateDate() _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - log.Fatalf("failed to write to time.csv %s", err) + return err } } @@ -179,21 +223,28 @@ func timeField() { date = generateDate() _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - log.Fatalf("failed to write to time.csv %s", err) + return err } } } + err1 := writer.Flush() + if err1 != nil { + return err1 + } - writer.Flush() - csvFile.Close() + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func optInField() { +func optInField() error { csvFile, err := os.Create("optin.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) @@ -202,7 +253,7 @@ func optInField() { for i := 0; i < totalRecords; i++ { - if rand.Intn(20) == 1 { //odds that optRandom == 1 are approx 5% + if rand.Intn(20) == 1 { optString = "1" } else { optString = "0" @@ -210,7 +261,7 @@ func optInField() { _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to optin.csv %s", err) + return err } if rand.Intn(10) == 1 { @@ -218,32 +269,40 @@ func optInField() { optString = "1" _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to optin.csv %s", err) + return err } } else { optString = "0" _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to optin.csv %s", err) + return err } } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } -func countryField() { +func countryField() error { csvFile, err := os.Create("country.csv") if err != nil { - log.Fatalf("failed to create file %s", err) + return err } writer := bufio.NewWriter(csvFile) - //populate countries + // populate countries r := rand.New(rand.NewSource(rand.Int63())) s := 1.01 v := 1.01 @@ -257,7 +316,7 @@ func countryField() { country := countryList[value] _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to country.csv %s", err) + return err } if rand.Intn(10) == 1 { @@ -265,22 +324,30 @@ func countryField() { country = countryList[value] _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") if err != nil { - log.Fatalf("failed to write to country.csv %s", err) + return err } } } } - writer.Flush() - csvFile.Close() + err1 := writer.Flush() + if err1 != nil { + return err1 + } + + err2 := csvFile.Close() + if err2 != nil { + return err2 + } + return nil } func generateDate() string { // generates a random date in format YYYY-MM-DDT00:00 // does not include leap days or populate the hour/minute time - year := strconv.Itoa(rand.Intn(20) + 2002) //year range 2002-2021 + year := strconv.Itoa(rand.Intn(20) + 2002) // year range 2002-2021 month := rand.Intn(12) + 1 day := rand.Intn(30) + 1 From f4a40e79b976512113993c1b3e371624e01ce041 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 12:08:24 -0600 Subject: [PATCH 174/222] final cleanup --- cmd/random-query/main.go | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index 0fcd71ce9..bef9a92c2 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -42,22 +42,23 @@ import ( // RandomQueryConfig type RandomQueryConfig struct { - HostPort string - TreeDepth int - QueryCount int - Verbose bool - NumRuns int - TimeFromArg string - TimeToArg string - TimeFrom time.Time - TimeTo time.Time - TimeRange int64 - Index string - QPM int - Seed int - SrcFile string - Duration time.Duration - Target vegeta.Target + HostPort string + TreeDepth int + QueryCount int + Verbose bool + GenerateOnly bool + NumRuns int + TimeFromArg string + TimeToArg string + TimeFrom time.Time + TimeTo time.Time + TimeRange int64 + Index string + QPM int + Seed int + SrcFile string + Duration time.Duration + Target vegeta.Target IndexMap map[string]*Features @@ -441,13 +442,6 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci f = &Features{} cfg.IndexMap[index] = f } - vprint.VV("Add: %s %v %v %v", field, min, max, scale) - /* if min.Scale != scale || max.Scale != scale { - - PanicOn(fmt.Sprintf("scale error; %v:%v min scale %d, max scale %d, field scale %d, assumed they'd be equal", - index, field, min.Scale, max.Scale, scale)) - } - */ effectiveRange := uint64(max.Value) - uint64(min.Value) + 1 // if you have INT64_MAX and INT64_MIN, effectiveRange is 1<<64, which From 5c5e16af33a7aeb1fb3f889ecbe72334b9dcb8ad Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Fri, 19 Nov 2021 12:26:23 -0600 Subject: [PATCH 175/222] added a generate only flag --- cmd/random-query/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go index bef9a92c2..e9dce0354 100644 --- a/cmd/random-query/main.go +++ b/cmd/random-query/main.go @@ -114,6 +114,7 @@ func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.Index, "index", "i", "index to run queries against") fs.IntVar(&cfg.QPM, "qpm", 10, "number of current requests per minute to simulate, default 10") fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") + fs.BoolVar(&cfg.GenerateOnly, "generate-only", false, "only generate do not run package") fs.StringVar(&cfg.TimeFromArg, "time.from", defaultStartTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.TimeToArg, "time.to", defaultEndTime.Format(time.RFC3339), "starting time for time fields (format: 2006-01-02T15:04:05Z07:00)") fs.StringVar(&cfg.SrcFile, "query-file", "", "use pql contained in this file for query batch instead of generating") @@ -390,6 +391,10 @@ func (cfg *RandomQueryConfig) buildPayload() error { Query: request.String(), } vprint.VV("%v", request.String()) + if cfg.GenerateOnly { + // just output the PQL and exit + os.Exit(0) + } payload, err := proto.Marshal(req) if err != nil { return errors.Wrap(err, "marshaling request to protobuf") From a02a8a94b36c2ad62edc89c4de02cc85db17edd8 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 19 Nov 2021 12:39:37 -0600 Subject: [PATCH 176/222] remove unused .cloudbuild directory (was for GCP CI stuff) --- .cloudbuild/Dockerfile | 23 ----- .cloudbuild/NOTICE | 109 -------------------- .cloudbuild/cloudbuild.yml | 202 ------------------------------------- 3 files changed, 334 deletions(-) delete mode 100644 .cloudbuild/Dockerfile delete mode 100644 .cloudbuild/NOTICE delete mode 100644 .cloudbuild/cloudbuild.yml diff --git a/.cloudbuild/Dockerfile b/.cloudbuild/Dockerfile deleted file mode 100644 index fac3bf2fd..000000000 --- a/.cloudbuild/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM alpine:3.14.2 - -LABEL maintainer "dev@molecula.com" -LABEL org.opencontainers.image.authors="dev@molecula.com" - -RUN apk add --no-cache curl jq - -COPY NOTICE /NOTICE -ARG SHORT_SHA - -COPY .cloudbuild/featurebase_linux_amd64 / - -RUN mv /featurebase_linux_amd64 /pilosa -RUN chmod ugo+x /pilosa -EXPOSE 10101 -VOLUME /data - -ENV PILOSA_DATA_DIR /data -ENV PILOSA_BIND 0.0.0.0:10101 -ENV PILOSA_BIND_GRPC 0.0.0.0:20101 - -ENTRYPOINT ["/pilosa"] -CMD ["server"] diff --git a/.cloudbuild/NOTICE b/.cloudbuild/NOTICE deleted file mode 100644 index 5d88ea9c0..000000000 --- a/.cloudbuild/NOTICE +++ /dev/null @@ -1,109 +0,0 @@ -Software license -================ - -Copyright (C) 2017-2021 Molecula Corp. All rights reserved. - -Third-party software licenses -============================= - -The file /pilosa/lru/lru.go contains a redistribution of lru -(github.com/golang/groupcache/lru); the license follows: - - Copyright 2013 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -The file /roaring/btree.go contains a modified redistribution of b -(https://github.com/cznic/b); the license follows: - - Copyright (c) 2014 The b Authors. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - * Neither the names of the authors nor the names of the - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The file /server/tlsconfig.go contains a modified redistribution of bridge -(https://github.com/robustirc/bridge); the license follows: - - Copyright © 2014-2015 The RobustIRC Authors. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of RobustIRC nor the names of contributors may be used - to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The files /logger/filewriter.go and /logger/filewriter_test.go contain a modified redistribution of reopen (github.com/client9/reopen); the license follows: - - The MIT License (MIT) - - Copyright (c) 2015 Nick Galbreath - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. diff --git a/.cloudbuild/cloudbuild.yml b/.cloudbuild/cloudbuild.yml deleted file mode 100644 index 4ec21187a..000000000 --- a/.cloudbuild/cloudbuild.yml +++ /dev/null @@ -1,202 +0,0 @@ -steps: - - name: gcr.io/cloud-builders/yarn - args: - - install - dir: lattice/ - id: Install Lattice - waitFor: - - '-' - - name: gcr.io/cloud-builders/yarn - args: - - build - dir: lattice/ - id: Build Lattice - waitFor: - - Install Lattice - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - tar - - '-c' - - '-z' - - '-v' - - '-f' - - lattice.tar.gz - - lattice/build - id: Package Lattice Build - waitFor: - - Build Lattice - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - mv - - lattice.tar.gz - - /cache/ - waitFor: - - Package Lattice Build - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - go - - test - - '-covermode=atomic' - - '-coverprofile=/cache/coverage.out' - id: Generate code coverage report - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - '-c' - - | - go test -json > /cache/test-report.out - id: Generate test report - waitFor: - - '-' - entrypoint: bash - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - go - - tool - - cover - - '-html' - - /cache/coverage.out - - '-o' - - /cache/coverage.html - id: Calculate Coverage - waitFor: - - Generate code coverage report - - name: 'sonarsource/sonar-scanner-cli:4.6' - args: - - '-Dsonar.projectKey=molecula_featurebase' - - '-Dsonar.organization=molecula' - - '-Dsonar.sources=.' - - '-Dsonar.host.url=https://sonarcloud.io' - - '-Dsonar.go.coverage.reportPaths=/cache/coverage.out' - - '-Dsonar.go.tests.reportPaths=/cache/test-report.out' - id: SonarCloud - waitFor: - - Calculate Coverage - secretEnv: - - SONAR_TOKEN - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - args: - - jfrog - - config - - add - - molecula - - '--interactive=false' - - '--artifactory-url=https://molecula.jfrog.io/' - - '--access-token=$$JFROG_API_TOKEN' - secretEnv: - - JFROG_API_TOKEN - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=linux - - GOARCH=amd64 - args: - - go - - build - - '-o' - - /cache/featurebase_linux_amd64 - - ./cmd/featurebase - id: Build for Linux amd64 - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=darwin - - GOARCH=amd64 - args: - - go - - build - - '-o' - - /cache/featurebase_darwin_amd64 - - ./cmd/featurebase - id: Build for Darwin amd64 - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=linux - - GOARCH=arm64 - args: - - go - - build - - '-o' - - /cache/featurebase_linux_arm64 - - ./cmd/featurebase - id: Build for Linux arm64 - waitFor: - - '-' - - name: 'us.gcr.io/featurebase-pipeline-dev/featurebase:0.3' - env: - - GOOS=darwin - - GOARCH=arm64 - args: - - go - - build - - '-o' - - /cache/featurebase_darwin_arm64 - - ./cmd/featurebase - id: Build for Darwin arm64 - waitFor: - - '-' - - name: gcr.io/cloud-builders/gsutil - args: - - '-c' - - > - gsutil -m cp -r '/cache/*' - "gs://molecula-pipeline/featurebase/dev/$SHORT_SHA/" - id: Upload Binaries by Short SHA - waitFor: - - Package Lattice Build - - Calculate Coverage - - Build for Linux amd64 - - Build for Darwin amd64 - - Build for Linux arm64 - - Build for Darwin arm64 - entrypoint: bash - - name: gcr.io/cloud-builders/gsutil - args: - - '-c' - - > - gsutil -m cp -r '/cache/*' - "gs://molecula-pipeline/featurebase/dev/$BRANCH_NAME/" - id: Upload Binaries by Branch Name - waitFor: - - Package Lattice Build - - Calculate Coverage - - Build for Linux amd64 - - Build for Darwin amd64 - - Build for Linux arm64 - - Build for Darwin arm64 - entrypoint: bash - - name: gcr.io/cloud-builders/docker - args: - - '-c' - - > - cp -v /cache/* .cloudbuild/ && docker build --build-arg - SHORT_SHA=$SHORT_SHA -t - northamerica-northeast1-docker.pkg.dev/featurebase-pipeline-dev/featurebase/main:$SHORT_SHA - -f .cloudbuild/Dockerfile . - id: Build Linux amd Container - entrypoint: bash - - name: gcr.io/cloud-builders/docker - args: - - push - - >- - northamerica-northeast1-docker.pkg.dev/featurebase-pipeline-dev/featurebase/main:$SHORT_SHA - waitFor: - - Build Linux amd Container -timeout: 1200s -logsBucket: 'gs://molecula-pipeline/logs' -options: - volumes: - - name: data - path: /cache - pool: - name: >- - projects/featurebase-pipeline-dev/locations/us-central1/workerPools/test-pool-00 -availableSecrets: - secretManager: - - versionName: projects/903994035405/secrets/jfrog-admin-token/versions/1 - env: JFROG_API_TOKEN - - versionName: projects/903994035405/secrets/sonarcloud-featurebase-token/versions/1 - env: SONAR_TOKEN From c83170f1f025a8dd33d6b7d6c0370ac40f3884fc Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 19 Nov 2021 13:16:48 -0600 Subject: [PATCH 177/222] decrease sleep time --- .gitlab/.gitlab-ci.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 0dda19409..0dc7a60c5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -205,7 +205,7 @@ deploy node for linux amd64: INSTANCE: "t3a.large" SECURITY_GROUP: "" SUBNET_ID: "" - PUBLIC_IP: "" + IP: "" PROFILE: "default" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: @@ -229,11 +229,8 @@ deploy node for linux amd64: - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - echo $INSTANCE_ID > linux-amd64-instance.txt - - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - - PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - - echo $PUBLIC_IP - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$PUBLIC_IP:. + - while [ -z "$IP" ]; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); sleep 5; done + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: - job: build for linux amd64 From 4f5f27f5138962d922e5bcfc611310d5ebc2f3ea Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Fri, 19 Nov 2021 11:47:32 -0800 Subject: [PATCH 178/222] make error handling best --- qa/simulacraData/simulacra_data.go | 93 +++++++++++++++--------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index f5741201a..14f40b42f 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -24,6 +24,8 @@ import ( "math/rand" "os" "strconv" + + "github.com/pkg/errors" ) var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "ALB", "AND", "ANT", "ARE", "ARG", @@ -48,7 +50,7 @@ const totalRecords int = 200000000 func main() { err0 := ageField() - if err0 != nil { + if err0 == nil { fmt.Printf("unable to generate age field: %v", err0) } @@ -78,9 +80,9 @@ func main() { } func ageField() error { - csvFile, err := os.Create("age.csv") - if err != nil { - return err + csvFile, err0 := os.Create("age.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create age.csv") } writer := bufio.NewWriter(csvFile) @@ -89,13 +91,14 @@ func ageField() error { if rand.Intn(100) <= 20 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to age.csv") } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to age.csv") + } } @@ -103,20 +106,20 @@ func ageField() error { } err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close age.csv") } return nil } func ipField() error { - csvFile, err := os.Create("ip.csv") - if err != nil { - return err + csvFile, err0 := os.Create("ip.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create ip.csv") } writer := bufio.NewWriter(csvFile) @@ -124,34 +127,34 @@ func ipField() error { for i := 0; i < totalRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to ip.csv") } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to ip.csv") } } } err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close ip.csv") } return nil } func arbIdField() error { - csvFile, err := os.Create("identifier.csv") - if err != nil { - return err + csvFile, err0 := os.Create("identifier.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create identifier.csv") } writer := bufio.NewWriter(csvFile) @@ -159,34 +162,34 @@ func arbIdField() error { for i := 0; i < totalRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to identifier.csv") } if rand.Intn(10) == 1 { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to identifier.csv") } } } err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close identifier.csv") } return nil } func timeField() error { - csvFile, err := os.Create("time.csv") - if err != nil { - return err + csvFile, err0 := os.Create("time.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create time.csv") } writer := bufio.NewWriter(csvFile) @@ -206,14 +209,14 @@ func timeField() error { _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to time.csv") } if rand.Intn(5) == 1 { date = generateDate() _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to time.csv") } } @@ -223,28 +226,28 @@ func timeField() error { date = generateDate() _, err := writer.WriteString(urlString + "," + strconv.Itoa(i) + "," + date + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to time.csv") } } } err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close time.csv") } return nil } func optInField() error { - csvFile, err := os.Create("optin.csv") - if err != nil { - return err + csvFile, err0 := os.Create("optin.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create optin.csv") } writer := bufio.NewWriter(csvFile) @@ -261,7 +264,7 @@ func optInField() error { _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to optin.csv") } if rand.Intn(10) == 1 { @@ -269,13 +272,13 @@ func optInField() error { optString = "1" _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to optin.csv") } } else { optString = "0" _, err := writer.WriteString(optString + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to optin.csv") } } } @@ -283,21 +286,21 @@ func optInField() error { err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close optin.csv") } return nil } func countryField() error { - csvFile, err := os.Create("country.csv") - if err != nil { - return err + csvFile, err0 := os.Create("country.csv") + if err0 != nil { + return errors.Wrap(err0, "unable to create country.csv") } writer := bufio.NewWriter(csvFile) @@ -316,7 +319,7 @@ func countryField() error { country := countryList[value] _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to country.csv") } if rand.Intn(10) == 1 { @@ -324,7 +327,7 @@ func countryField() error { country = countryList[value] _, err := writer.WriteString(country + "," + strconv.Itoa(i) + "\n") if err != nil { - return err + return errors.Wrap(err, "unable to write to country.csv") } } } @@ -333,12 +336,12 @@ func countryField() error { err1 := writer.Flush() if err1 != nil { - return err1 + return errors.Wrap(err1, "unable to flush writer") } err2 := csvFile.Close() if err2 != nil { - return err2 + return errors.Wrap(err2, "unable to close country.csv") } return nil From e4745c1d779440984fd5fc2b0214afdf212d90df Mon Sep 17 00:00:00 2001 From: reesporte Date: Fri, 19 Nov 2021 14:32:08 -0600 Subject: [PATCH 179/222] add test for timestamp Min/Max on multinode clusters --- executor_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/executor_test.go b/executor_test.go index b6c4b993a..1c1c9747a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -8292,3 +8292,44 @@ func TestToRows(t *testing.T) { } } + +// TestMinMaxTimestampVariableNode tests Min() and Max() queries on +// timestamp values on a clusters of varying size. +func TestMinMaxTimestampVariableNode(t *testing.T) { + for i := 1; i < 4; i++ { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + // separate function so we can defer closing the cluster + // and so it looks prettier + MinMaxTimestampNodeTester(t, i) + }) + } +} + +// MinMaxTimestampNodeTester tests Min() and Max() queries on +// timestamp values on a cluster with `numNodes` nodes. +// fails if the min or max values are not correct +func MinMaxTimestampNodeTester(t *testing.T, numNodes int) { + t.Helper() + index := "test_index" + field := "ts" + c := test.MustRunCluster(t, numNodes) + defer c.Close() + + // create an index and timestamp field + c.CreateField(t, index, pilosa.IndexOptions{}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s")) + + // add some data + expected := "2010-01-02T12:32:00Z" + c.Query(t, index, "Set(10, ts=\""+expected+"\")") + + // run a query on the cluster + min := c.Query(t, index, "Min("+field+")").Results[0].(pilosa.ValCount).TimestampVal.Format(time.RFC3339Nano) + if min != expected { + t.Fatalf("incorrect min timestamp val. expected: %v, got %v\n", expected, min) + } + + max := c.Query(t, index, "Max("+field+")").Results[0].(pilosa.ValCount).TimestampVal.Format(time.RFC3339Nano) + if max != expected { + t.Fatalf("incorrect max timestamp val. expected: %v, got %v\n", expected, min) + } +} From 2cde034e8287a1169d3e6e47de5e03c0ef30a865 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Fri, 19 Nov 2021 14:40:37 -0600 Subject: [PATCH 180/222] testing reviewer's suggestion --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 0dc7a60c5..fddcb9be6 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -229,7 +229,7 @@ deploy node for linux amd64: - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - echo $INSTANCE_ID > linux-amd64-instance.txt - - while [ -z "$IP" ]; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); sleep 5; done + - for i in {0..24}; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); if [ -n "$IP" ]; then break; fi; sleep 5; done - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: From fa635874d4cee9999e67caff9b6b021d8c9787fc Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 22 Nov 2021 10:08:18 -0600 Subject: [PATCH 181/222] encode/decode timestamp val appropriately --- encoding/proto/proto.go | 25 ++-- pb/public.pb.go | 251 ++++++++++++++++++++++++---------------- pb/public.proto | 1 + 3 files changed, 168 insertions(+), 109 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index c5672e563..b121de978 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -19,7 +19,7 @@ import ( "time" "github.com/gogo/protobuf/proto" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" @@ -1637,11 +1637,17 @@ func (s Serializer) decodePairField(pb *pb.PairField) pilosa.PairField { } func (s Serializer) decodeValCount(pb *pb.ValCount) pilosa.ValCount { + var t time.Time + t, err := time.Parse(time.RFC3339Nano, pb.TimestampVal) + if err != nil { + t = time.Time{} + } return pilosa.ValCount{ - Val: pb.Val, - FloatVal: pb.FloatVal, - DecimalVal: s.decodeDecimalStruct(pb.DecimalVal), - Count: pb.Count, + Val: pb.Val, + FloatVal: pb.FloatVal, + DecimalVal: s.decodeDecimalStruct(pb.DecimalVal), + TimestampVal: t, + Count: pb.Count, } } @@ -1849,10 +1855,11 @@ func (s Serializer) encodePairField(p pilosa.PairField) *pb.PairField { func (s Serializer) encodeValCount(vc pilosa.ValCount) *pb.ValCount { return &pb.ValCount{ - Val: vc.Val, - FloatVal: vc.FloatVal, - DecimalVal: s.encodeDecimal(vc.DecimalVal), - Count: vc.Count, + Val: vc.Val, + FloatVal: vc.FloatVal, + DecimalVal: s.encodeDecimal(vc.DecimalVal), + Count: vc.Count, + TimestampVal: vc.TimestampVal.Format(time.RFC3339Nano), } } diff --git a/pb/public.pb.go b/pb/public.pb.go index ca1b84852..34c78e3d5 100644 --- a/pb/public.pb.go +++ b/pb/public.pb.go @@ -1168,6 +1168,7 @@ type ValCount struct { Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` FloatVal float64 `protobuf:"fixed64,3,opt,name=FloatVal,proto3" json:"FloatVal,omitempty"` DecimalVal *Decimal `protobuf:"bytes,4,opt,name=DecimalVal,proto3" json:"DecimalVal,omitempty"` + TimestampVal string `protobuf:"bytes,5,opt,name=TimestampVal,proto3" json:"TimestampVal,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1234,6 +1235,13 @@ func (m *ValCount) GetDecimalVal() *Decimal { return nil } +func (m *ValCount) GetTimestampVal() string { + if m != nil { + return m.TimestampVal + } + return "" +} + type Decimal struct { Value int64 `protobuf:"varint,1,opt,name=Value,proto3" json:"Value,omitempty"` Scale int64 `protobuf:"varint,2,opt,name=Scale,proto3" json:"Scale,omitempty"` @@ -2449,106 +2457,106 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1569 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdd, 0x6e, 0x1b, 0xd5, - 0x16, 0xce, 0xfc, 0xf8, 0x6f, 0xd9, 0x71, 0xd2, 0xdd, 0xb4, 0x67, 0x4e, 0x4f, 0xea, 0xe3, 0x8e, - 0x8e, 0x2a, 0xf7, 0x04, 0xa5, 0xc2, 0x40, 0x85, 0x2a, 0x01, 0x8a, 0xe3, 0x94, 0x8c, 0xda, 0xa4, - 0x65, 0x27, 0x04, 0x2e, 0xb8, 0x99, 0xd8, 0x1b, 0x77, 0xc4, 0xd8, 0x63, 0xc6, 0xe3, 0x3a, 0x16, - 0x0f, 0x00, 0x8f, 0xc0, 0x0b, 0xf0, 0x28, 0x08, 0xee, 0xe0, 0x92, 0x4b, 0x54, 0x5e, 0x04, 0xad, - 0xb5, 0xf7, 0xfc, 0xda, 0xad, 0xaa, 0x8a, 0xbb, 0x59, 0x3f, 0x7b, 0xfd, 0x7c, 0x6b, 0xed, 0xb5, - 0xb6, 0x0d, 0x8d, 0xe9, 0xfc, 0xd2, 0xf7, 0x06, 0xfb, 0xd3, 0x30, 0x88, 0x02, 0xa6, 0x4f, 0x2f, - 0xed, 0x25, 0x18, 0x3c, 0x58, 0x30, 0x0b, 0x2a, 0x87, 0x81, 0x3f, 0x1f, 0x4f, 0x66, 0x96, 0xd6, - 0x36, 0x3a, 0x26, 0x8f, 0x49, 0xc6, 0xc0, 0x7c, 0x2c, 0x96, 0x33, 0xcb, 0x68, 0x1b, 0x9d, 0x1a, - 0xa7, 0x6f, 0xd4, 0xe6, 0x81, 0x1b, 0x7a, 0x93, 0x91, 0x65, 0xb6, 0xb5, 0x4e, 0x83, 0xc7, 0x24, - 0xdb, 0x81, 0x92, 0x33, 0x19, 0x8a, 0x2b, 0xab, 0xd4, 0xd6, 0x3a, 0x35, 0x2e, 0x09, 0xe4, 0x3e, - 0xf2, 0x84, 0x3f, 0xb4, 0xca, 0x92, 0x4b, 0x84, 0xdd, 0x81, 0x1a, 0x0f, 0x16, 0x27, 0x6e, 0x14, - 0x7a, 0x57, 0xec, 0x3f, 0x60, 0xf2, 0x60, 0x21, 0xbd, 0xd7, 0xbb, 0x95, 0xfd, 0xe9, 0xe5, 0x3e, - 0x0f, 0x16, 0x9c, 0x98, 0xf6, 0x01, 0xd4, 0xce, 0xbc, 0xd1, 0x44, 0x0c, 0x31, 0xd4, 0x7f, 0x83, - 0xf1, 0x2c, 0x40, 0x45, 0x2d, 0xab, 0x88, 0x3c, 0x14, 0x9d, 0x8a, 0x91, 0xa5, 0x17, 0x44, 0xa7, - 0x62, 0x64, 0x7f, 0x08, 0x4d, 0x1e, 0x2c, 0x9c, 0xa1, 0x98, 0x44, 0xde, 0xd7, 0x9e, 0x08, 0x29, - 0xb1, 0xc4, 0xa3, 0x29, 0x1d, 0x25, 0xc9, 0xea, 0x69, 0xb2, 0xf6, 0x2d, 0x28, 0x3b, 0xfd, 0x27, - 0xde, 0x2c, 0x62, 0xdb, 0x60, 0x38, 0xfd, 0xf8, 0x00, 0x7e, 0xda, 0x87, 0x70, 0xed, 0xe8, 0x2a, - 0x0a, 0xdd, 0x41, 0x24, 0x86, 0x4e, 0x5f, 0x42, 0xc6, 0x9a, 0xa0, 0x3b, 0x7d, 0x8a, 0xcf, 0xe4, - 0xba, 0xd3, 0x67, 0x2d, 0x30, 0x2f, 0x5c, 0x5f, 0x1a, 0xad, 0x77, 0x01, 0xc3, 0x92, 0x06, 0x39, - 0xf1, 0xed, 0xaf, 0x72, 0x46, 0x14, 0x1e, 0x37, 0xa1, 0x4c, 0x28, 0x49, 0x77, 0x35, 0xae, 0x28, - 0x76, 0x3f, 0x2d, 0x94, 0xb4, 0x77, 0x03, 0xed, 0xad, 0x04, 0x91, 0xd4, 0xcf, 0xbe, 0x0d, 0x95, - 0xc7, 0x62, 0x49, 0xf1, 0xc7, 0xd9, 0x69, 0x99, 0xec, 0x7e, 0xd3, 0xe0, 0x7a, 0x72, 0xfa, 0xdc, - 0xbd, 0xf4, 0xc5, 0x85, 0xeb, 0xcf, 0x05, 0x6b, 0xc5, 0xb9, 0x6a, 0xf9, 0x98, 0x8f, 0x37, 0x28, - 0x73, 0x76, 0x27, 0x41, 0x0a, 0x15, 0xea, 0xa8, 0xa0, 0xdc, 0x1c, 0x6f, 0xa8, 0x2e, 0xd9, 0x85, - 0x6a, 0xef, 0xcc, 0x21, 0x73, 0x96, 0xd1, 0xd6, 0x3a, 0xc6, 0xf1, 0x06, 0x4f, 0x38, 0xec, 0x16, - 0x54, 0x4e, 0xe6, 0x91, 0xb8, 0x72, 0xfa, 0xd4, 0x43, 0xe6, 0xf1, 0x06, 0x8f, 0x19, 0x78, 0x92, - 0x3e, 0x1f, 0x8b, 0xa5, 0x6c, 0x24, 0x3c, 0x19, 0x73, 0xd8, 0x0e, 0x98, 0xbd, 0x20, 0xf0, 0xa9, - 0x99, 0xaa, 0xe8, 0x0d, 0xa9, 0x5e, 0x05, 0x4a, 0x64, 0xd8, 0xbe, 0x82, 0x9d, 0x7c, 0x42, 0xaa, - 0x2c, 0x0c, 0x0c, 0xb4, 0xa7, 0x29, 0x7b, 0x48, 0xb0, 0x6d, 0x2a, 0x95, 0xae, 0xfc, 0x63, 0xb1, - 0xee, 0x43, 0x99, 0xcc, 0xc8, 0x86, 0xaf, 0x77, 0xff, 0x95, 0x83, 0x37, 0x05, 0x88, 0x2b, 0xb5, - 0x5e, 0x8d, 0xf0, 0x7d, 0x1a, 0x3a, 0x7d, 0xfb, 0xa3, 0x22, 0x94, 0x54, 0x33, 0x84, 0xfd, 0xd4, - 0x1d, 0x0b, 0xe9, 0x99, 0xd3, 0x37, 0xf2, 0xce, 0x97, 0x53, 0x41, 0xae, 0x6b, 0x9c, 0xbe, 0xed, - 0x39, 0x34, 0xf3, 0xc7, 0x31, 0x98, 0x4c, 0x13, 0xac, 0x0d, 0x86, 0xe4, 0x49, 0x77, 0x74, 0x8b, - 0xdd, 0x61, 0xad, 0x9e, 0x28, 0x36, 0xc8, 0xc7, 0x60, 0x3e, 0x73, 0xbd, 0x70, 0xa5, 0x6d, 0xb7, - 0x25, 0x5e, 0x06, 0x45, 0x68, 0x48, 0xe0, 0x4b, 0x87, 0xc1, 0x7c, 0x12, 0x49, 0xc0, 0xb8, 0x24, - 0xec, 0x4f, 0xa0, 0x86, 0xe7, 0x65, 0xae, 0xbb, 0xd2, 0x98, 0xea, 0x9b, 0x2a, 0x7a, 0x47, 0x9a, - 0x4b, 0x17, 0xc9, 0x1c, 0xd0, 0xb3, 0x73, 0xa0, 0x07, 0x80, 0xd2, 0x99, 0xb4, 0xd0, 0x82, 0x12, - 0x51, 0x2a, 0xe5, 0xd4, 0x84, 0x64, 0xbf, 0xc2, 0xc6, 0x6d, 0x9c, 0x3b, 0xd1, 0x83, 0xf7, 0x51, - 0x2c, 0x3b, 0x0e, 0x23, 0x30, 0xb8, 0xea, 0x89, 0x00, 0xaa, 0x12, 0xa8, 0x60, 0x91, 0x1a, 0xd0, - 0x32, 0x06, 0x90, 0x8b, 0xf3, 0xa1, 0x1f, 0xe7, 0x46, 0x04, 0xde, 0x42, 0x1e, 0x2c, 0x52, 0x18, - 0x14, 0xc5, 0xfe, 0x1b, 0x7b, 0x31, 0x29, 0xcf, 0x1a, 0xdd, 0x0f, 0xf4, 0x1f, 0x3b, 0xfc, 0x12, - 0xe0, 0xd3, 0x30, 0x98, 0x4f, 0x09, 0x22, 0x66, 0x43, 0x89, 0x28, 0x95, 0x53, 0x03, 0xd5, 0xe3, - 0x78, 0xb8, 0x14, 0xad, 0x07, 0x17, 0x8b, 0x70, 0x30, 0x1a, 0xc9, 0xeb, 0xc3, 0xf1, 0xd3, 0xfe, - 0x0e, 0xaa, 0x17, 0xae, 0x9f, 0x48, 0x2f, 0x5c, 0x5f, 0xa5, 0x8a, 0x9f, 0x79, 0x2b, 0x46, 0x6c, - 0xe5, 0x16, 0x54, 0x1f, 0xf9, 0x81, 0x1b, 0xa1, 0x32, 0x9a, 0xd2, 0x78, 0x42, 0xb3, 0x3d, 0x80, - 0xbe, 0x18, 0x78, 0x63, 0xd7, 0x47, 0xa9, 0x99, 0x5e, 0x67, 0xc5, 0xe5, 0x19, 0xb1, 0xfd, 0x01, - 0x54, 0x14, 0xb5, 0x1e, 0x68, 0xe4, 0x9e, 0x0d, 0x5c, 0x5f, 0xc4, 0xfe, 0x89, 0xb0, 0x7f, 0xd6, - 0xa0, 0xf1, 0xd9, 0x5c, 0x84, 0x4b, 0x2e, 0xbe, 0x9d, 0x8b, 0x59, 0x84, 0x6a, 0x44, 0xc7, 0x35, - 0x20, 0x02, 0xd1, 0x3e, 0x7b, 0xee, 0x86, 0x43, 0xd9, 0xbc, 0x26, 0x57, 0x14, 0x55, 0x41, 0x8c, - 0x83, 0x48, 0xd0, 0x30, 0xa8, 0x72, 0x45, 0xb1, 0x3d, 0x68, 0x1c, 0x8d, 0x2f, 0xc5, 0x70, 0x28, - 0x86, 0x7d, 0x37, 0x72, 0xad, 0x6a, 0x7e, 0x77, 0xe4, 0x84, 0xec, 0x7f, 0xb0, 0xf9, 0x2c, 0x14, - 0xe7, 0xa1, 0x3b, 0x99, 0xf9, 0x6e, 0x24, 0x86, 0x56, 0x8d, 0x6c, 0xe5, 0x99, 0x6c, 0x17, 0x6a, - 0x27, 0xee, 0xd5, 0x89, 0x18, 0x07, 0xe1, 0xd2, 0x02, 0xca, 0x21, 0x65, 0xd8, 0x4f, 0x60, 0x53, - 0xa5, 0x31, 0x9b, 0x06, 0x93, 0x99, 0xc0, 0x02, 0x1c, 0x85, 0xa1, 0xca, 0x02, 0x3f, 0xd9, 0x3d, - 0xa8, 0x70, 0x31, 0x9b, 0xfb, 0x51, 0x7c, 0x03, 0xb7, 0x30, 0x9c, 0xf8, 0xd4, 0xdc, 0x8f, 0x78, - 0x2c, 0xb7, 0x7f, 0x2a, 0x41, 0x3d, 0x23, 0x48, 0x66, 0x02, 0xce, 0xb5, 0x4d, 0x39, 0x13, 0x70, - 0xa3, 0xf1, 0x60, 0xb1, 0xb2, 0xec, 0xb0, 0x8f, 0x1b, 0xa0, 0x9d, 0xaa, 0x66, 0xd1, 0x4e, 0xd3, - 0x6b, 0x63, 0xac, 0xbf, 0x36, 0xb8, 0xe0, 0x9f, 0xbb, 0x93, 0x91, 0x18, 0x52, 0x8d, 0xab, 0x3c, - 0x26, 0x59, 0x27, 0x6d, 0x28, 0xc2, 0x57, 0xf5, 0x67, 0xcc, 0xe3, 0x69, 0xbb, 0xc9, 0xdb, 0x80, - 0x6b, 0xa1, 0x22, 0xeb, 0x23, 0x29, 0xf6, 0x00, 0x9a, 0x4f, 0xfd, 0x61, 0xda, 0xef, 0x33, 0x55, - 0x89, 0x26, 0xda, 0x49, 0xd9, 0xbc, 0xa0, 0xc5, 0x1e, 0x16, 0x77, 0x32, 0xd5, 0xa4, 0xde, 0x65, - 0x2a, 0xcf, 0x8c, 0x84, 0x17, 0xb7, 0xf7, 0x5e, 0xe6, 0x49, 0x40, 0x85, 0xaa, 0x77, 0x37, 0xf1, - 0x58, 0xc2, 0xe4, 0x99, 0x27, 0xc3, 0x7e, 0x76, 0xc2, 0x58, 0x75, 0xd2, 0x6e, 0xc6, 0x08, 0x49, - 0x2e, 0xcf, 0xce, 0xa0, 0xbd, 0xcc, 0x48, 0xb3, 0x1a, 0xa9, 0xf1, 0x84, 0xc9, 0x33, 0x23, 0xef, - 0x70, 0xcd, 0xfa, 0xb6, 0x36, 0xe9, 0x50, 0x71, 0x37, 0x4b, 0x21, 0x5f, 0xb3, 0xee, 0x1f, 0x16, - 0x67, 0xbf, 0xd5, 0x4c, 0xa1, 0xc8, 0x4b, 0x78, 0x71, 0x4b, 0xec, 0x65, 0xde, 0x51, 0xd6, 0x56, - 0x1a, 0x6d, 0xc2, 0xe4, 0x99, 0x77, 0xd6, 0xbb, 0x50, 0xcf, 0x16, 0x6a, 0x9b, 0xd4, 0xb7, 0xf2, - 0x85, 0x9a, 0xf1, 0xac, 0x8e, 0xfd, 0x8b, 0x0e, 0x9b, 0xce, 0x78, 0x1a, 0x84, 0x51, 0xe6, 0xfa, - 0xca, 0x57, 0x9e, 0xb6, 0xf6, 0x95, 0xa7, 0x17, 0x06, 0x2b, 0x5d, 0x63, 0x1a, 0x3c, 0x26, 0x97, - 0x44, 0xa6, 0x95, 0xcc, 0x5c, 0x2b, 0xed, 0x42, 0x4d, 0xee, 0x25, 0x14, 0x95, 0x48, 0x94, 0x32, - 0xe4, 0xbb, 0x73, 0x41, 0xef, 0x8e, 0x0a, 0xbd, 0x61, 0x62, 0x92, 0xb5, 0x00, 0xa4, 0x1a, 0x09, - 0xab, 0x24, 0xcc, 0x70, 0x50, 0x7e, 0xee, 0x8d, 0xc5, 0x2c, 0x72, 0xc7, 0xd3, 0x99, 0x55, 0x6e, - 0x1b, 0x1d, 0x83, 0x67, 0x38, 0xec, 0x2e, 0x34, 0x29, 0x89, 0xc3, 0x50, 0xe0, 0x1c, 0x38, 0x88, - 0xa8, 0x15, 0x0d, 0x5e, 0xe0, 0xa2, 0x1e, 0xa5, 0x95, 0xea, 0xc9, 0x21, 0x51, 0xe0, 0xd2, 0x1c, - 0xf6, 0x85, 0x1b, 0x52, 0xb3, 0x55, 0xb9, 0x24, 0xec, 0x3f, 0x74, 0x60, 0x12, 0x49, 0xf9, 0x86, - 0xf8, 0xc7, 0xe0, 0x7c, 0x3d, 0x6c, 0x79, 0x70, 0x2a, 0x2b, 0xe0, 0xdc, 0x4c, 0xde, 0x3c, 0x12, - 0x18, 0x45, 0xb1, 0x36, 0xd4, 0xe3, 0x35, 0x81, 0x42, 0x44, 0x55, 0xe3, 0x59, 0x16, 0xb3, 0xa1, - 0x71, 0x16, 0xe1, 0xc3, 0x5f, 0xa9, 0xd4, 0xc8, 0x76, 0x8e, 0xb7, 0x06, 0x5a, 0x78, 0x43, 0x68, - 0xeb, 0xaf, 0x87, 0xb6, 0x91, 0x85, 0xf6, 0x7b, 0x0d, 0x1a, 0x07, 0x51, 0x30, 0xf6, 0x06, 0x5c, - 0x0c, 0x82, 0x70, 0xf8, 0x6a, 0x50, 0x25, 0x7c, 0x7a, 0x16, 0xbe, 0x0e, 0x18, 0xce, 0x8b, 0x50, - 0x8d, 0xce, 0x9b, 0xb4, 0xcc, 0x57, 0xaa, 0xc4, 0x51, 0x85, 0xdd, 0x01, 0xdd, 0x09, 0xa9, 0x67, - 0xeb, 0xdd, 0x6b, 0xa9, 0x62, 0xac, 0xa3, 0x3b, 0xa1, 0xfd, 0x0e, 0xec, 0xc8, 0x40, 0x62, 0x91, - 0xda, 0x15, 0x3b, 0x50, 0x3a, 0x0a, 0xc3, 0x20, 0xde, 0x16, 0x92, 0xc0, 0xd7, 0x6a, 0xb2, 0x7e, - 0xb0, 0x18, 0x6f, 0xd3, 0x13, 0xeb, 0x7e, 0xa2, 0xb5, 0xa1, 0x7e, 0x1a, 0x44, 0x5f, 0x84, 0x5e, - 0x44, 0xd3, 0x44, 0xce, 0xfc, 0x2c, 0xcb, 0xbe, 0x07, 0x37, 0x0a, 0x9e, 0xd3, 0xa5, 0x86, 0x6d, - 0x64, 0xa4, 0x3f, 0x73, 0xce, 0xe0, 0x7a, 0xa2, 0xea, 0xf4, 0xdf, 0x2a, 0xc6, 0x55, 0xa3, 0xff, - 0xcf, 0x64, 0x4e, 0x46, 0x95, 0xfb, 0x35, 0xd9, 0xd8, 0x3d, 0xb0, 0x14, 0x9a, 0xf2, 0x77, 0xa6, - 0x8a, 0xe0, 0xc2, 0x13, 0x8b, 0x57, 0x3d, 0xaf, 0xe9, 0x45, 0xa0, 0xd3, 0xaf, 0x53, 0xfa, 0xb6, - 0x7f, 0xd0, 0x61, 0x67, 0x9d, 0x91, 0xb4, 0xa1, 0xb4, 0x4c, 0x43, 0xb1, 0x2e, 0x94, 0x5e, 0x78, - 0x62, 0x11, 0xaf, 0xf1, 0xdd, 0x4c, 0xb1, 0x57, 0x62, 0xe0, 0x52, 0x15, 0x2f, 0xd2, 0xc1, 0x20, - 0xf2, 0x82, 0x49, 0xfc, 0x5c, 0x94, 0x14, 0x7a, 0xe8, 0xf9, 0xc1, 0xe0, 0x1b, 0xf9, 0x4b, 0x87, - 0x4b, 0x62, 0xcd, 0xc5, 0x28, 0xbd, 0xe1, 0xc5, 0x28, 0xaf, 0xbd, 0x18, 0x1d, 0xd8, 0xfa, 0x7c, - 0x3a, 0x74, 0x23, 0x71, 0x74, 0xe5, 0xcd, 0x22, 0x31, 0x19, 0x08, 0xab, 0x42, 0x19, 0x15, 0xd9, - 0xf6, 0x59, 0x6e, 0x09, 0xe0, 0xf4, 0x38, 0x18, 0x8d, 0x42, 0x31, 0x72, 0xa3, 0x18, 0xc6, 0x94, - 0xc1, 0xee, 0x42, 0x99, 0x94, 0x63, 0x24, 0x8a, 0x5b, 0x5d, 0x49, 0x7b, 0xdb, 0xbf, 0xbe, 0x6c, - 0x69, 0xbf, 0xbf, 0x6c, 0x69, 0x7f, 0xbe, 0x6c, 0x69, 0x3f, 0xfe, 0xd5, 0xda, 0xb8, 0x2c, 0xd3, - 0xdf, 0x0c, 0xef, 0xfd, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x70, 0xbe, 0xda, 0x79, 0x76, 0x10, 0x00, - 0x00, + // 1582 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5f, 0x6f, 0x1b, 0xd5, + 0x12, 0xcf, 0xfe, 0xf1, 0xbf, 0xb1, 0xe3, 0xa4, 0xa7, 0x69, 0xef, 0xde, 0xde, 0xd4, 0xd7, 0x5d, + 0x5d, 0x55, 0xee, 0x0d, 0x4a, 0x85, 0x81, 0x0a, 0x55, 0x02, 0x14, 0xc7, 0x29, 0x59, 0xb5, 0x49, + 0xcb, 0x49, 0x08, 0x3c, 0xf0, 0xb2, 0xb1, 0x0f, 0xee, 0x8a, 0xb5, 0xd7, 0xac, 0xd7, 0x75, 0xfc, + 0x09, 0xe0, 0x23, 0xf0, 0xc6, 0x13, 0x1f, 0x05, 0xc1, 0x1b, 0x3c, 0xf2, 0x88, 0xca, 0x17, 0x41, + 0x33, 0xe7, 0xec, 0x5f, 0xbb, 0x55, 0x55, 0xf1, 0xb6, 0xf3, 0xe7, 0xcc, 0x99, 0xf9, 0xcd, 0x9c, + 0x99, 0xb1, 0xa1, 0x31, 0x9d, 0x5f, 0xfa, 0xde, 0x60, 0x7f, 0x1a, 0x06, 0x51, 0xc0, 0xf4, 0xe9, + 0xa5, 0xbd, 0x04, 0x83, 0x07, 0x0b, 0x66, 0x41, 0xe5, 0x30, 0xf0, 0xe7, 0xe3, 0xc9, 0xcc, 0xd2, + 0xda, 0x46, 0xc7, 0xe4, 0x31, 0xc9, 0x18, 0x98, 0x8f, 0xc5, 0x72, 0x66, 0x19, 0x6d, 0xa3, 0x53, + 0xe3, 0xf4, 0x8d, 0xda, 0x3c, 0x70, 0x43, 0x6f, 0x32, 0xb2, 0xcc, 0xb6, 0xd6, 0x69, 0xf0, 0x98, + 0x64, 0x3b, 0x50, 0x72, 0x26, 0x43, 0x71, 0x65, 0x95, 0xda, 0x5a, 0xa7, 0xc6, 0x25, 0x81, 0xdc, + 0x47, 0x9e, 0xf0, 0x87, 0x56, 0x59, 0x72, 0x89, 0xb0, 0x3b, 0x50, 0xe3, 0xc1, 0xe2, 0xc4, 0x8d, + 0x42, 0xef, 0x8a, 0xfd, 0x07, 0x4c, 0x1e, 0x2c, 0xe4, 0xed, 0xf5, 0x6e, 0x65, 0x7f, 0x7a, 0xb9, + 0xcf, 0x83, 0x05, 0x27, 0xa6, 0x7d, 0x00, 0xb5, 0x33, 0x6f, 0x34, 0x11, 0x43, 0x74, 0xf5, 0xdf, + 0x60, 0x3c, 0x0b, 0x50, 0x51, 0xcb, 0x2a, 0x22, 0x0f, 0x45, 0xa7, 0x62, 0x64, 0xe9, 0x05, 0xd1, + 0xa9, 0x18, 0xd9, 0x1f, 0x42, 0x93, 0x07, 0x0b, 0x67, 0x28, 0x26, 0x91, 0xf7, 0xb5, 0x27, 0x42, + 0x0a, 0x2c, 0xb9, 0xd1, 0x94, 0x17, 0x25, 0xc1, 0xea, 0x69, 0xb0, 0xf6, 0x2d, 0x28, 0x3b, 0xfd, + 0x27, 0xde, 0x2c, 0x62, 0xdb, 0x60, 0x38, 0xfd, 0xf8, 0x00, 0x7e, 0xda, 0x87, 0x70, 0xed, 0xe8, + 0x2a, 0x0a, 0xdd, 0x41, 0x24, 0x86, 0x4e, 0x5f, 0x42, 0xc6, 0x9a, 0xa0, 0x3b, 0x7d, 0xf2, 0xcf, + 0xe4, 0xba, 0xd3, 0x67, 0x2d, 0x30, 0x2f, 0x5c, 0x5f, 0x1a, 0xad, 0x77, 0x01, 0xdd, 0x92, 0x06, + 0x39, 0xf1, 0xed, 0xaf, 0x72, 0x46, 0x14, 0x1e, 0x37, 0xa1, 0x4c, 0x28, 0xc9, 0xeb, 0x6a, 0x5c, + 0x51, 0xec, 0x7e, 0x9a, 0x28, 0x69, 0xef, 0x06, 0xda, 0x5b, 0x71, 0x22, 0xc9, 0x9f, 0x7d, 0x1b, + 0x2a, 0x8f, 0xc5, 0x92, 0xfc, 0x8f, 0xa3, 0xd3, 0x32, 0xd1, 0xfd, 0xa6, 0xc1, 0xf5, 0xe4, 0xf4, + 0xb9, 0x7b, 0xe9, 0x8b, 0x0b, 0xd7, 0x9f, 0x0b, 0xd6, 0x8a, 0x63, 0xd5, 0xf2, 0x3e, 0x1f, 0x6f, + 0x50, 0xe4, 0xec, 0x4e, 0x82, 0x14, 0x2a, 0xd4, 0x51, 0x41, 0x5d, 0x73, 0xbc, 0xa1, 0xaa, 0x64, + 0x17, 0xaa, 0xbd, 0x33, 0x87, 0xcc, 0x59, 0x46, 0x5b, 0xeb, 0x18, 0xc7, 0x1b, 0x3c, 0xe1, 0xb0, + 0x5b, 0x50, 0x39, 0x99, 0x47, 0xe2, 0xca, 0xe9, 0x53, 0x0d, 0x99, 0xc7, 0x1b, 0x3c, 0x66, 0xe0, + 0x49, 0xfa, 0x7c, 0x2c, 0x96, 0xb2, 0x90, 0xf0, 0x64, 0xcc, 0x61, 0x3b, 0x60, 0xf6, 0x82, 0xc0, + 0xa7, 0x62, 0xaa, 0xe2, 0x6d, 0x48, 0xf5, 0x2a, 0x50, 0x22, 0xc3, 0xf6, 0x15, 0xec, 0xe4, 0x03, + 0x52, 0x69, 0x61, 0x60, 0xa0, 0x3d, 0x4d, 0xd9, 0x43, 0x82, 0x6d, 0x53, 0xaa, 0x74, 0x75, 0x3f, + 0x26, 0xeb, 0x3e, 0x94, 0xc9, 0x8c, 0x2c, 0xf8, 0x7a, 0xf7, 0x5f, 0x39, 0x78, 0x53, 0x80, 0xb8, + 0x52, 0xeb, 0xd5, 0x08, 0xdf, 0xa7, 0xa1, 0xd3, 0xb7, 0x3f, 0x2a, 0x42, 0x49, 0x39, 0x43, 0xd8, + 0x4f, 0xdd, 0xb1, 0x90, 0x37, 0x73, 0xfa, 0x46, 0xde, 0xf9, 0x72, 0x2a, 0xe8, 0xea, 0x1a, 0xa7, + 0x6f, 0x7b, 0x0e, 0xcd, 0xfc, 0x71, 0x74, 0x26, 0x53, 0x04, 0x6b, 0x9d, 0x21, 0x79, 0x52, 0x1d, + 0xdd, 0x62, 0x75, 0x58, 0xab, 0x27, 0x8a, 0x05, 0xf2, 0x31, 0x98, 0xcf, 0x5c, 0x2f, 0x5c, 0x29, + 0xdb, 0x6d, 0x89, 0x97, 0x41, 0x1e, 0x1a, 0x12, 0xf8, 0xd2, 0x61, 0x30, 0x9f, 0x44, 0x12, 0x30, + 0x2e, 0x09, 0xfb, 0x13, 0xa8, 0xe1, 0x79, 0x19, 0xeb, 0xae, 0x34, 0xa6, 0xea, 0xa6, 0x8a, 0xb7, + 0x23, 0xcd, 0xe5, 0x15, 0x49, 0x1f, 0xd0, 0xb3, 0x7d, 0xa0, 0x07, 0x80, 0xd2, 0x99, 0xb4, 0xd0, + 0x82, 0x12, 0x51, 0x2a, 0xe4, 0xd4, 0x84, 0x64, 0xbf, 0xc2, 0xc6, 0x6d, 0xec, 0x3b, 0xd1, 0x83, + 0xf7, 0x51, 0x2c, 0x2b, 0x0e, 0x3d, 0x30, 0xb8, 0xaa, 0x89, 0x00, 0xaa, 0x12, 0xa8, 0x60, 0x91, + 0x1a, 0xd0, 0x32, 0x06, 0x90, 0x8b, 0xfd, 0xa1, 0x1f, 0xc7, 0x46, 0x04, 0xbe, 0x42, 0x1e, 0x2c, + 0x52, 0x18, 0x14, 0xc5, 0xfe, 0x1b, 0xdf, 0x62, 0x52, 0x9c, 0x35, 0x7a, 0x1f, 0x78, 0x7f, 0x7c, + 0xe1, 0x97, 0x00, 0x9f, 0x86, 0xc1, 0x7c, 0x4a, 0x10, 0x31, 0x1b, 0x4a, 0x44, 0xa9, 0x98, 0x1a, + 0xa8, 0x1e, 0xfb, 0xc3, 0xa5, 0x68, 0x3d, 0xb8, 0x98, 0x84, 0x83, 0xd1, 0x48, 0x3e, 0x1f, 0x8e, + 0x9f, 0xf6, 0x8f, 0x1a, 0x54, 0x2f, 0x5c, 0x3f, 0x11, 0x5f, 0xb8, 0xbe, 0x8a, 0x15, 0x3f, 0xf3, + 0x66, 0x8c, 0xd8, 0xcc, 0x2d, 0xa8, 0x3e, 0xf2, 0x03, 0x37, 0x42, 0x65, 0xb4, 0xa5, 0xf1, 0x84, + 0x66, 0x7b, 0x00, 0x7d, 0x31, 0xf0, 0xc6, 0xae, 0x8f, 0x52, 0x33, 0x7d, 0xcf, 0x8a, 0xcb, 0x33, + 0x62, 0x66, 0x43, 0xe3, 0xdc, 0x1b, 0x8b, 0x59, 0xe4, 0x8e, 0xa7, 0xa8, 0x2e, 0xdb, 0x7c, 0x8e, + 0x67, 0x7f, 0x00, 0x15, 0x75, 0x62, 0x7d, 0x36, 0x90, 0x7b, 0x36, 0x70, 0x7d, 0x11, 0xfb, 0x48, + 0x84, 0xfd, 0xb3, 0x06, 0x8d, 0xcf, 0xe6, 0x22, 0x5c, 0x72, 0xf1, 0xed, 0x5c, 0xcc, 0x22, 0x54, + 0x23, 0x3a, 0x4e, 0x14, 0x11, 0x98, 0x92, 0xb3, 0xe7, 0x6e, 0x38, 0x94, 0x15, 0x6e, 0x72, 0x45, + 0x51, 0xaa, 0xc4, 0x38, 0x88, 0x04, 0xf9, 0x54, 0xe5, 0x8a, 0x62, 0x7b, 0xd0, 0x38, 0x1a, 0x5f, + 0x8a, 0xe1, 0x50, 0x0c, 0xfb, 0x6e, 0xe4, 0x5a, 0xd5, 0xfc, 0x80, 0xc9, 0x09, 0xd9, 0xff, 0x60, + 0xf3, 0x59, 0x28, 0xce, 0x43, 0x77, 0x32, 0xf3, 0xdd, 0x48, 0x0c, 0xad, 0x1a, 0xd9, 0xca, 0x33, + 0xd9, 0x2e, 0xd4, 0x4e, 0xdc, 0xab, 0x13, 0x31, 0x0e, 0xc2, 0xa5, 0x05, 0x14, 0x43, 0xca, 0xb0, + 0x9f, 0xc0, 0xa6, 0x0a, 0x63, 0x36, 0x0d, 0x26, 0x33, 0x81, 0x49, 0x3a, 0x0a, 0x43, 0x15, 0x05, + 0x7e, 0xb2, 0x7b, 0x50, 0xe1, 0x62, 0x36, 0xf7, 0xa3, 0xf8, 0x99, 0x6e, 0xa1, 0x3b, 0xf1, 0xa9, + 0xb9, 0x1f, 0xf1, 0x58, 0x6e, 0xff, 0x54, 0x82, 0x7a, 0x46, 0x90, 0x34, 0x0e, 0x6c, 0x7e, 0x9b, + 0xb2, 0x71, 0xe0, 0xd8, 0xe3, 0xc1, 0x62, 0x65, 0x22, 0x62, 0xb1, 0x37, 0x40, 0x3b, 0x55, 0x15, + 0xa5, 0x9d, 0xa6, 0x6f, 0xcb, 0x58, 0xff, 0xb6, 0x70, 0x0b, 0x78, 0xee, 0x4e, 0x46, 0x62, 0x48, + 0x75, 0x50, 0xe5, 0x31, 0xc9, 0x3a, 0x69, 0xd1, 0x11, 0xbe, 0xaa, 0x88, 0x63, 0x1e, 0x4f, 0x4b, + 0x52, 0x3e, 0x19, 0x9c, 0x1d, 0x15, 0x99, 0x1f, 0x49, 0xb1, 0x07, 0xd0, 0x7c, 0xea, 0x0f, 0xd3, + 0x47, 0x31, 0x53, 0x99, 0x68, 0xa2, 0x9d, 0x94, 0xcd, 0x0b, 0x5a, 0xec, 0x61, 0x71, 0x70, 0x53, + 0x4e, 0xea, 0x5d, 0xa6, 0xe2, 0xcc, 0x48, 0x78, 0x71, 0xc4, 0xef, 0x65, 0xf6, 0x06, 0x4a, 0x54, + 0xbd, 0xbb, 0x89, 0xc7, 0x12, 0x26, 0xcf, 0xec, 0x15, 0xfb, 0xd9, 0x36, 0x64, 0xd5, 0x49, 0xbb, + 0x19, 0x23, 0x24, 0xb9, 0x3c, 0xdb, 0xa8, 0xf6, 0x32, 0x7d, 0xcf, 0x6a, 0xa4, 0xc6, 0x13, 0x26, + 0xcf, 0xf4, 0xc5, 0xc3, 0x35, 0x33, 0xde, 0xda, 0xa4, 0x43, 0xc5, 0x01, 0x2e, 0x85, 0x7c, 0xcd, + 0x4e, 0xf0, 0xb0, 0x38, 0x20, 0xac, 0x66, 0x0a, 0x45, 0x5e, 0xc2, 0x8b, 0xa3, 0x64, 0x2f, 0xb3, + 0x6c, 0x59, 0x5b, 0xa9, 0xb7, 0x09, 0x93, 0x67, 0x96, 0xb1, 0x77, 0xa1, 0x9e, 0x4d, 0xd4, 0x36, + 0xa9, 0x6f, 0xe5, 0x13, 0x35, 0xe3, 0x59, 0x1d, 0xfb, 0x17, 0x1d, 0x36, 0x9d, 0xf1, 0x34, 0x08, + 0xa3, 0xcc, 0xf3, 0x95, 0xab, 0xa0, 0xb6, 0x76, 0x15, 0xd4, 0x0b, 0xdd, 0x97, 0x9e, 0x31, 0x35, + 0x27, 0x93, 0x4b, 0x22, 0x53, 0x4a, 0x66, 0xae, 0x94, 0x76, 0xa1, 0x26, 0x87, 0x17, 0x8a, 0x4a, + 0x24, 0x4a, 0x19, 0x72, 0x39, 0x5d, 0xd0, 0x72, 0x52, 0xa1, 0x45, 0x27, 0x26, 0x59, 0x0b, 0x40, + 0xaa, 0x91, 0xb0, 0x4a, 0xc2, 0x0c, 0x07, 0xe5, 0x49, 0x23, 0x9b, 0x59, 0xe5, 0xb6, 0xd1, 0x31, + 0x78, 0x86, 0xc3, 0xee, 0x42, 0x93, 0x82, 0x38, 0x0c, 0x05, 0xf6, 0x81, 0x83, 0x88, 0x4a, 0xd1, + 0xe0, 0x05, 0x2e, 0xea, 0x51, 0x58, 0xa9, 0x9e, 0x6c, 0x12, 0x05, 0x2e, 0xf5, 0x6a, 0x5f, 0xb8, + 0x21, 0x15, 0x5b, 0x95, 0x4b, 0xc2, 0xfe, 0x43, 0x07, 0x26, 0x91, 0x94, 0x8b, 0xc6, 0x3f, 0x06, + 0xe7, 0xeb, 0x61, 0xcb, 0x83, 0x53, 0x59, 0x01, 0xe7, 0x66, 0xb2, 0x18, 0x49, 0x60, 0x14, 0xc5, + 0xda, 0x50, 0x8f, 0x47, 0x09, 0x0a, 0x11, 0x55, 0x8d, 0x67, 0x59, 0x38, 0x33, 0xce, 0x22, 0xfc, + 0x75, 0xa0, 0x54, 0x6a, 0x64, 0x3b, 0xc7, 0x5b, 0x03, 0x2d, 0xbc, 0x21, 0xb4, 0xf5, 0xd7, 0x43, + 0xdb, 0xc8, 0x42, 0xfb, 0x9d, 0x06, 0x8d, 0x83, 0x28, 0x18, 0x7b, 0x03, 0x2e, 0x06, 0x41, 0x38, + 0x7c, 0x35, 0xa8, 0x12, 0x3e, 0x3d, 0x0b, 0x5f, 0x07, 0x0c, 0xe7, 0x45, 0xa8, 0x5a, 0xe7, 0x4d, + 0x9a, 0xf8, 0x2b, 0x59, 0xe2, 0xa8, 0xc2, 0xee, 0x80, 0xee, 0x84, 0x54, 0xb3, 0xf5, 0xee, 0xb5, + 0x54, 0x31, 0xd6, 0xd1, 0x9d, 0xd0, 0x7e, 0x07, 0x76, 0xa4, 0x23, 0xb1, 0x48, 0xcd, 0x8a, 0x1d, + 0x28, 0x1d, 0x85, 0x61, 0x10, 0x4f, 0x0b, 0x49, 0xe0, 0x4a, 0x9b, 0x8c, 0x1f, 0x4c, 0xc6, 0xdb, + 0xd4, 0xc4, 0xba, 0xdf, 0x71, 0x6d, 0xa8, 0x9f, 0x06, 0xd1, 0x17, 0xa1, 0x17, 0x51, 0x37, 0x91, + 0x3d, 0x3f, 0xcb, 0xb2, 0xef, 0xc1, 0x8d, 0xc2, 0xcd, 0xe9, 0x50, 0xc3, 0x32, 0x32, 0xd2, 0xdf, + 0x42, 0x67, 0x70, 0x3d, 0x51, 0x75, 0xfa, 0x6f, 0xe5, 0xe3, 0xaa, 0xd1, 0xff, 0x67, 0x22, 0x27, + 0xa3, 0xea, 0xfa, 0x35, 0xd1, 0xd8, 0x3d, 0xb0, 0x14, 0x9a, 0xf2, 0xc7, 0xa8, 0xf2, 0xe0, 0xc2, + 0x13, 0x8b, 0x57, 0xed, 0xe0, 0xb4, 0x11, 0xe8, 0xf4, 0x13, 0x96, 0xbe, 0xed, 0xef, 0x75, 0xd8, + 0x59, 0x67, 0x24, 0x2d, 0x28, 0x2d, 0x53, 0x50, 0xac, 0x0b, 0xa5, 0x17, 0x9e, 0x58, 0xc4, 0x63, + 0x7c, 0x37, 0x93, 0xec, 0x15, 0x1f, 0xb8, 0x54, 0xc5, 0x87, 0x74, 0x30, 0x88, 0xbc, 0x60, 0x12, + 0xef, 0x94, 0x92, 0xc2, 0x1b, 0x7a, 0x7e, 0x30, 0xf8, 0x46, 0xfe, 0x1c, 0xe2, 0x92, 0x58, 0xf3, + 0x30, 0x4a, 0x6f, 0xf8, 0x30, 0xca, 0x6b, 0x1f, 0x46, 0x07, 0xb6, 0x3e, 0x9f, 0x0e, 0xdd, 0x48, + 0x1c, 0x5d, 0x79, 0xb3, 0x48, 0x4c, 0x06, 0xc2, 0xaa, 0x50, 0x44, 0x45, 0xb6, 0x7d, 0x96, 0x1b, + 0x02, 0xd8, 0x3d, 0x0e, 0x46, 0xa3, 0x50, 0x8c, 0xdc, 0x28, 0x86, 0x31, 0x65, 0xb0, 0xbb, 0x50, + 0x26, 0xe5, 0x18, 0x89, 0xe2, 0x54, 0x57, 0xd2, 0xde, 0xf6, 0xaf, 0x2f, 0x5b, 0xda, 0xef, 0x2f, + 0x5b, 0xda, 0x9f, 0x2f, 0x5b, 0xda, 0x0f, 0x7f, 0xb5, 0x36, 0x2e, 0xcb, 0xf4, 0x5f, 0xc4, 0x7b, + 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x88, 0x20, 0x3d, 0x60, 0x9b, 0x10, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -3557,6 +3565,13 @@ func (m *ValCount) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.TimestampVal) > 0 { + i -= len(m.TimestampVal) + copy(dAtA[i:], m.TimestampVal) + i = encodeVarintPublic(dAtA, i, uint64(len(m.TimestampVal))) + i-- + dAtA[i] = 0x2a + } if m.DecimalVal != nil { { size, err := m.DecimalVal.MarshalToSizedBuffer(dAtA[:i]) @@ -5224,6 +5239,10 @@ func (m *ValCount) Size() (n int) { l = m.DecimalVal.Size() n += 1 + l + sovPublic(uint64(l)) } + l = len(m.TimestampVal) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8213,6 +8232,38 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimestampVal", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimestampVal = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/pb/public.proto b/pb/public.proto index 70935c647..d86a80d5d 100644 --- a/pb/public.proto +++ b/pb/public.proto @@ -109,6 +109,7 @@ message ValCount { int64 Count = 2; double FloatVal = 3; Decimal DecimalVal = 4; + string TimestampVal = 5; } message Decimal { From 55d35365c6d9fd7c9c98a4f2872c29c784a6f620 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 22 Nov 2021 10:53:42 -0600 Subject: [PATCH 182/222] allow failure for ec2 node job --- .gitlab/.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index fddcb9be6..b48a7c9cd 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -237,6 +237,7 @@ deploy node for linux amd64: artifacts: paths: - linux-amd64-instance.txt + allow_failure: true terminate node for linux amd64: stage: integration @@ -253,3 +254,4 @@ terminate node for linux amd64: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: deploy node for linux amd64 + From a32a217dbdc7443490729f3cfb6f74f7a3987f28 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 22 Nov 2021 12:10:42 -0600 Subject: [PATCH 183/222] test error condition --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index b48a7c9cd..7946dc655 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -232,8 +232,8 @@ deploy node for linux amd64: - for i in {0..24}; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); if [ -n "$IP" ]; then break; fi; sleep 5; done - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - needs: - - job: build for linux amd64 + # needs: + # - job: build for linux amd64 artifacts: paths: - linux-amd64-instance.txt From 97c4ee670b5f4e2925c207902c66f9446666e144 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 10:43:13 -0800 Subject: [PATCH 184/222] add tests --- qa/simulacraData/simulacra_data_test.go | 94 +++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 qa/simulacraData/simulacra_data_test.go diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go new file mode 100644 index 000000000..54e785b0c --- /dev/null +++ b/qa/simulacraData/simulacra_data_test.go @@ -0,0 +1,94 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import ( + "fmt" + "testing" +) + +func TestAge(t *testing.T) { + err := ageField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestIP(t *testing.T) { + err := ipField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestIndentifer(t *testing.T) { + err := arbIdField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestOptIn(t *testing.T) { + err := optInField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestCountry(t *testing.T) { + err := countryField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestTime(t *testing.T) { + err := timeField() + if err != nil { + fmt.Printf("%v", err) + } +} + +func TestGenerateDate(t *testing.T) { + testDate := generateDate() + if testDate == "" { + fmt.Printf("error generating date") + } +} + +// func TestReadAge(t *testing.T) { +// csvfile, err := os.Open("age.csv") +// if err != nil { +// fmt.Printf("%v", err) +// } + +// r := csv.NewReader(csvfile) +// records, err := r.ReadAll() +// if err != nil { +// fmt.Printf("%v", err) +// } + +// for i := 0; i < len(records); i++ { + +// age, err := strconv.Atoi(records[i][1]) +// if err != nil { +// fmt.Printf("%v", err) +// } + +// if age > 100 || age < 13 { +// fmt.Printf("age outside of allowed range") +// } +// fmt.Println(age) +// } +// } From 2385fda2593297593b89b0249cffcb5e87e54b1a Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 10:45:12 -0800 Subject: [PATCH 185/222] add tests, remove commented code --- qa/simulacraData/simulacra_data_test.go | 26 ------------------------- 1 file changed, 26 deletions(-) diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go index 54e785b0c..0620e54bc 100644 --- a/qa/simulacraData/simulacra_data_test.go +++ b/qa/simulacraData/simulacra_data_test.go @@ -66,29 +66,3 @@ func TestGenerateDate(t *testing.T) { fmt.Printf("error generating date") } } - -// func TestReadAge(t *testing.T) { -// csvfile, err := os.Open("age.csv") -// if err != nil { -// fmt.Printf("%v", err) -// } - -// r := csv.NewReader(csvfile) -// records, err := r.ReadAll() -// if err != nil { -// fmt.Printf("%v", err) -// } - -// for i := 0; i < len(records); i++ { - -// age, err := strconv.Atoi(records[i][1]) -// if err != nil { -// fmt.Printf("%v", err) -// } - -// if age > 100 || age < 13 { -// fmt.Printf("age outside of allowed range") -// } -// fmt.Println(age) -// } -// } From 06be6c0981524ce19a9f6bc5af39ffd307698474 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 11:06:38 -0800 Subject: [PATCH 186/222] use log.Fatal in lieu of fmt.Print --- qa/simulacraData/simulacra_data.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 14f40b42f..656c02211 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -21,6 +21,7 @@ package main import ( "bufio" "fmt" + "log" "math/rand" "os" "strconv" @@ -51,31 +52,31 @@ const totalRecords int = 200000000 func main() { err0 := ageField() if err0 == nil { - fmt.Printf("unable to generate age field: %v", err0) + log.Fatalf("unable to generate age field: %v", err0) } err1 := ipField() if err1 != nil { - fmt.Printf("unable to generate IPfield: %v", err1) + log.Fatalf("unable to generate IPfield: %v", err1) } err2 := arbIdField() if err2 != nil { - fmt.Printf("unable to generate indentifier field: %v", err2) + log.Fatalf("unable to generate indentifier field: %v", err2) } err3 := optInField() if err3 != nil { - fmt.Printf("unable to generate opt in field: %v", err3) + log.Fatalf("unable to generate opt in field: %v", err3) } err4 := countryField() if err4 != nil { - fmt.Printf("unable to generate country field: %v", err4) + log.Fatalf("unable to generate country field: %v", err4) } err5 := timeField() if err5 != nil { - fmt.Printf("unable to generate time field: %v", err5) + log.Fatalf("unable to generate time field: %v", err5) } } From f7fc380bcda748840c32d52a06f97d42c7c32dfa Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 22 Nov 2021 13:14:58 -0600 Subject: [PATCH 187/222] split aws commands to allow for better error handling --- .gitlab/.gitlab-ci.yml | 51 +++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 7946dc655..dea63374f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,7 +196,7 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -# deploy EC2 instance and upload featurebase binary +# deploy EC2 instance and save instance ID deploy node for linux amd64: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -205,9 +205,37 @@ deploy node for linux amd64: INSTANCE: "t3a.large" SECURITY_GROUP: "" SUBNET_ID: "" + PROFILE: "default" + AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY + before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem + - chmod 400 gitlab-featurebase-dev.pem + script: + - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) + - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) + - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) + - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json + - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') + - echo $INSTANCE_ID > linux-amd64-instance.txt + artifacts: + paths: + - linux-amd64-instance.txt + needs: + - job: build for linux amd64 + +# scp featurebase binary, config and service file, and configure featurebase +initiate featurebase linux amd64 in node: + stage: integration + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: IP: "" PROFILE: "default" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY + INSTANCE_ID: "" before_script: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY @@ -221,24 +249,15 @@ deploy node for linux amd64: - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - apt update && apt -y install jq + - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://.gitlab/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - - echo $INSTANCE_ID > linux-amd64-instance.txt - for i in {0..24}; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); if [ -n "$IP" ]; then break; fi; sleep 5; done - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 - # needs: - # - job: build for linux amd64 - artifacts: - paths: - - linux-amd64-instance.txt - allow_failure: true + needs: + - job: deploy node for linux amd64 +# terminate EC2 instance terminate node for linux amd64: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest @@ -249,9 +268,9 @@ terminate node for linux amd64: - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - INSTANCE_ID=$(cat linux-amd64-instance.txt) - - echo $INSTANCE_ID script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - job: deploy node for linux amd64 - + - job: initiate featurebase linux amd64 in node + when: always \ No newline at end of file From c51a2de67244184b6adf39afe9b1a4f839e23efe Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Mon, 22 Nov 2021 14:10:34 -0600 Subject: [PATCH 188/222] fixed dependency error --- .gitlab/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index dea63374f..ba793e62e 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -255,6 +255,7 @@ initiate featurebase linux amd64 in node: - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$IP:. - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: + - job: build for linux amd64 - job: deploy node for linux amd64 # terminate EC2 instance From 88fd6b1a9a6ecdd176e5651244c40cbd16d0d1c0 Mon Sep 17 00:00:00 2001 From: reesporte Date: Mon, 22 Nov 2021 15:06:00 -0600 Subject: [PATCH 189/222] remove helper status --- executor_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/executor_test.go b/executor_test.go index 1c1c9747a..e4924887e 100644 --- a/executor_test.go +++ b/executor_test.go @@ -8309,7 +8309,6 @@ func TestMinMaxTimestampVariableNode(t *testing.T) { // timestamp values on a cluster with `numNodes` nodes. // fails if the min or max values are not correct func MinMaxTimestampNodeTester(t *testing.T, numNodes int) { - t.Helper() index := "test_index" field := "ts" c := test.MustRunCluster(t, numNodes) From 49289f2237dafce5ad7fa96e3a7e2b68286e2895 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 13:53:41 -0800 Subject: [PATCH 190/222] replace fmt.Print with t.Fatal, and improve date generator --- qa/simulacraData/simulacra_data.go | 28 +++++++------------------ qa/simulacraData/simulacra_data_test.go | 15 +++++++------ 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 656c02211..1e1656a95 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -25,6 +25,7 @@ import ( "math/rand" "os" "strconv" + "time" "github.com/pkg/errors" ) @@ -349,28 +350,13 @@ func countryField() error { } func generateDate() string { - // generates a random date in format YYYY-MM-DDT00:00 - // does not include leap days or populate the hour/minute time - year := strconv.Itoa(rand.Intn(20) + 2002) // year range 2002-2021 - month := rand.Intn(12) + 1 - day := rand.Intn(30) + 1 + // Nov 22, 2021 = 1637616960 + // twenty years = 631138520 + // date min = 1006478440 + date := rand.Int63n(631138520) + 1006478440 - if month == 2 && day >= 29 { - day = 28 - } - if month == 4 && day == 31 { - day = 30 - } - if month == 6 && day == 31 { - day = 30 - } - if month == 11 && day == 31 { - day = 30 - } - - monthFix := fmt.Sprintf("%02d", month) - dayFix := fmt.Sprintf("%02d", day) - YMDstring := year + "-" + monthFix + "-" + dayFix + "T00:00" + t := time.Unix(date, 0) + YMDstring := t.Format("2006-01-02T15:04") return YMDstring } diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go index 0620e54bc..17dc8b623 100644 --- a/qa/simulacraData/simulacra_data_test.go +++ b/qa/simulacraData/simulacra_data_test.go @@ -14,55 +14,54 @@ package main import ( - "fmt" "testing" ) func TestAge(t *testing.T) { err := ageField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestIP(t *testing.T) { err := ipField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestIndentifer(t *testing.T) { err := arbIdField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestOptIn(t *testing.T) { err := optInField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestCountry(t *testing.T) { err := countryField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestTime(t *testing.T) { err := timeField() if err != nil { - fmt.Printf("%v", err) + t.Fatalf("%v", err) } } func TestGenerateDate(t *testing.T) { testDate := generateDate() if testDate == "" { - fmt.Printf("error generating date") + t.Fatalf("error generating date") } } From 50e7cda07fd6f83264575d4d7c686189f45ebf18 Mon Sep 17 00:00:00 2001 From: kcrodgers24 Date: Mon, 22 Nov 2021 14:53:17 -0800 Subject: [PATCH 191/222] modify functions to accept a requested num of records --- qa/simulacraData/simulacra_data.go | 57 +++++++++++-------------- qa/simulacraData/simulacra_data_test.go | 14 +++--- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/qa/simulacraData/simulacra_data.go b/qa/simulacraData/simulacra_data.go index 1e1656a95..e94c619d8 100644 --- a/qa/simulacraData/simulacra_data.go +++ b/qa/simulacraData/simulacra_data.go @@ -51,37 +51,32 @@ var countryList [246]string = [...]string{"ABW", "AFG", "AGO", "AIA", "ALA", "AL const totalRecords int = 200000000 func main() { - err0 := ageField() - if err0 == nil { - log.Fatalf("unable to generate age field: %v", err0) + if err := GenerateAgeField(totalRecords); err != nil { + log.Fatalf("unable to generate age field: %v", err) } - err1 := ipField() - if err1 != nil { - log.Fatalf("unable to generate IPfield: %v", err1) + if err := GenerateIPField(totalRecords); err != nil { + log.Fatalf("unable to generate IP field: %v", err) } - err2 := arbIdField() - if err2 != nil { - log.Fatalf("unable to generate indentifier field: %v", err2) + if err := GenerateArbIdField(totalRecords); err != nil { + log.Fatalf("unable to generate indentifier field: %v", err) } - err3 := optInField() - if err3 != nil { - log.Fatalf("unable to generate opt in field: %v", err3) - } - err4 := countryField() - if err4 != nil { - log.Fatalf("unable to generate country field: %v", err4) + if err := GenerateOptInField(totalRecords); err != nil { + log.Fatalf("unable to generate opt in field: %v", err) } - err5 := timeField() - if err5 != nil { - log.Fatalf("unable to generate time field: %v", err5) + if err := GenerateCountryField(totalRecords); err != nil { + log.Fatalf("unable to generate country field: %v", err) + } + + if err := GenerateTimeField(totalRecords); err != nil { + log.Fatalf("unable to generate time field: %v", err) } } -func ageField() error { +func GenerateAgeField(requestedRecords int) error { csvFile, err0 := os.Create("age.csv") if err0 != nil { return errors.Wrap(err0, "unable to create age.csv") @@ -89,7 +84,7 @@ func ageField() error { writer := bufio.NewWriter(csvFile) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { if rand.Intn(100) <= 20 { _, err := writer.WriteString(strconv.Itoa(i) + "," + strconv.Itoa(rand.Intn(88)+13) + "\n") if err != nil { @@ -118,7 +113,7 @@ func ageField() error { return nil } -func ipField() error { +func GenerateIPField(requestedRecords int) error { csvFile, err0 := os.Create("ip.csv") if err0 != nil { return errors.Wrap(err0, "unable to create ip.csv") @@ -126,7 +121,7 @@ func ipField() error { writer := bufio.NewWriter(csvFile) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int31()) + "\n") if err != nil { return errors.Wrap(err, "unable to write to ip.csv") @@ -153,7 +148,7 @@ func ipField() error { } -func arbIdField() error { +func GenerateArbIdField(requestedRecords int) error { csvFile, err0 := os.Create("identifier.csv") if err0 != nil { return errors.Wrap(err0, "unable to create identifier.csv") @@ -161,7 +156,7 @@ func arbIdField() error { writer := bufio.NewWriter(csvFile) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { _, err := writer.WriteString(strconv.Itoa(i) + "," + fmt.Sprint(rand.Int63()) + "\n") if err != nil { return errors.Wrap(err, "unable to write to identifier.csv") @@ -188,7 +183,7 @@ func arbIdField() error { } -func timeField() error { +func GenerateTimeField(requestedRecords int) error { csvFile, err0 := os.Create("time.csv") if err0 != nil { return errors.Wrap(err0, "unable to create time.csv") @@ -203,7 +198,7 @@ func timeField() error { distro := rand.NewZipf(r, s, v, imax) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { value := distro.Uint64() urlString := "https://www.test.com/" + strconv.FormatUint(value, 10) @@ -246,7 +241,7 @@ func timeField() error { } -func optInField() error { +func GenerateOptInField(requestedRecords int) error { csvFile, err0 := os.Create("optin.csv") if err0 != nil { return errors.Wrap(err0, "unable to create optin.csv") @@ -256,7 +251,7 @@ func optInField() error { var optString string - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { if rand.Intn(20) == 1 { optString = "1" @@ -299,7 +294,7 @@ func optInField() error { } -func countryField() error { +func GenerateCountryField(requestedRecords int) error { csvFile, err0 := os.Create("country.csv") if err0 != nil { return errors.Wrap(err0, "unable to create country.csv") @@ -315,7 +310,7 @@ func countryField() error { distro := rand.NewZipf(r, s, v, imax) - for i := 0; i < totalRecords; i++ { + for i := 0; i < requestedRecords; i++ { if rand.Intn(2) == 1 { value := distro.Uint64() country := countryList[value] diff --git a/qa/simulacraData/simulacra_data_test.go b/qa/simulacraData/simulacra_data_test.go index 17dc8b623..38462f5e5 100644 --- a/qa/simulacraData/simulacra_data_test.go +++ b/qa/simulacraData/simulacra_data_test.go @@ -17,43 +17,45 @@ import ( "testing" ) +const testRecords int = 1000 + func TestAge(t *testing.T) { - err := ageField() + err := GenerateAgeField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestIP(t *testing.T) { - err := ipField() + err := GenerateIPField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestIndentifer(t *testing.T) { - err := arbIdField() + err := GenerateArbIdField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestOptIn(t *testing.T) { - err := optInField() + err := GenerateOptInField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestCountry(t *testing.T) { - err := countryField() + err := GenerateCountryField(testRecords) if err != nil { t.Fatalf("%v", err) } } func TestTime(t *testing.T) { - err := timeField() + err := GenerateTimeField(testRecords) if err != nil { t.Fatalf("%v", err) } From 14911bfdff3b94a667a671c436eb4dbae2a34a04 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 23 Nov 2021 10:00:08 -0600 Subject: [PATCH 192/222] Perform sanity check only if we have the storage it runs on SanityCheckMapping is specific to roaring bitmaps stored in-memory, if we have an RBF backend, we shouldn't even try it, it'll just panic. This implies that, in whatever circumstance we were hitting this, we were getting an error back from the backend. We still need to address that error, but to do that we need to know what it was, which we don't if we panic. --- fragment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index 674f2da65..ad9283551 100644 --- a/fragment.go +++ b/fragment.go @@ -2485,7 +2485,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 err = doFunc() } - if err != nil { + if err != nil && f.storage != nil { // we got an error. it's possible that the error indicates that something went wrong. mappedIn, mappedOut, unmappedIn, errs, e2 := f.storage.SanityCheckMapping(f.currdata.from, f.currdata.to) if errs != 0 { From fbf44635722ae22a961b44e3dccd1d42c8b16161 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 11:08:14 -0600 Subject: [PATCH 193/222] remove duplicate code in multiple stages --- .gitlab/.gitlab-ci.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 4fe913596..9448a0311 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -221,12 +221,6 @@ deploy node for linux amd64: - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - echo $INSTANCE_ID > linux-amd64-instance.txt - - sleep 120 # Need to wait for the EC2 instance to launch and run the initialization commands passed through user-data - - PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - - echo $PUBLIC_IP - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ec2-user@$PUBLIC_IP:. - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$PUBLIC_IP:. - - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: - job: build for linux amd64 artifacts: @@ -260,8 +254,8 @@ initiate featurebase linux amd64 in node: - INSTANCE_ID=$(cat linux-amd64-instance.txt) script: - for i in {0..24}; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); if [ -n "$IP" ]; then break; fi; sleep 5; done - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 .gitlab/featurebase.conf .gitlab/featurebase.service ec2-user@$IP:. - - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://.gitlab/configureFeatureBase.json --profile $PROFILE --region us-east-2 + - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. + - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region us-east-2 needs: - job: build for linux amd64 - job: deploy node for linux amd64 From 7e8cbdeb53d891248f5e3e74204db57b6eebb3bb Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 12:09:51 -0600 Subject: [PATCH 194/222] updated a comment to trigger the pipeline for another test --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 9448a0311..c2161df43 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,7 +196,7 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -# deploy EC2 instance and save instance ID +# deploy EC2 instance and save instance ID to a text file deploy node for linux amd64: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest From 692f2f8cb46f48a2241ac0918afcdec3fcd26c58 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 13:29:56 -0600 Subject: [PATCH 195/222] updated dependency of jobs and made instanceId a global variable --- .gitlab/.gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c2161df43..80796aa3c 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,6 +12,7 @@ include: - .go/pkg/mod/ variables: GOVERSION: "1.16.9" + INSTANCE_ID_FILE: ./linux-amd64-instance.txt stages: - lint @@ -220,12 +221,12 @@ deploy node for linux amd64: - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - - echo $INSTANCE_ID > linux-amd64-instance.txt + - echo $INSTANCE_ID > $INSTANCE_ID_FILE needs: - job: build for linux amd64 artifacts: paths: - - linux-amd64-instance.txt + - $INSTANCE_ID_FILE needs: - job: build for linux amd64 @@ -251,7 +252,7 @@ initiate featurebase linux amd64 in node: - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - INSTANCE_ID=$(cat linux-amd64-instance.txt) + - INSTANCE_ID=$(cat $INSTANCE_ID_FILE) script: - for i in {0..24}; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); if [ -n "$IP" ]; then break; fi; sleep 5; done - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. @@ -270,10 +271,9 @@ terminate node for linux amd64: - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region "us-east-2" - - INSTANCE_ID=$(cat linux-amd64-instance.txt) + - INSTANCE_ID=$(cat $INSTANCE_ID_FILE) script: - aws ec2 terminate-instances --instance-ids $INSTANCE_ID needs: - - job: deploy node for linux amd64 - job: initiate featurebase linux amd64 in node when: always From cce48a2638c13b28de0c3e9e7790219ea3f91d9e Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 15:24:47 -0600 Subject: [PATCH 196/222] created a shell script for deplooying node and handling error conditions --- .gitlab/.gitlab-ci.yml | 82 ++-------------------------------- qa/scripts/deployNode.sh | 96 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 78 deletions(-) create mode 100755 qa/scripts/deployNode.sh diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 80796aa3c..656e4e4d5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -12,13 +12,13 @@ include: - .go/pkg/mod/ variables: GOVERSION: "1.16.9" - INSTANCE_ID_FILE: ./linux-amd64-instance.txt stages: + - integration - lint - test - build - - integration + #before_script: #- echo "before_script" @@ -115,7 +115,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: build + stage: integration image: golang:1.16.10 script: - rm -r lattice @@ -201,79 +201,5 @@ build container fb: deploy node for linux amd64: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - AMI: "" - INSTANCE: "t3a.large" - SECURITY_GROUP: "" - SUBNET_ID: "" - PROFILE: "default" - AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY - before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - - aws configure set aws_profile $PROFILE - - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - - chmod 400 gitlab-featurebase-dev.pem script: - - AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) - - SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) - - SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) - - aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') - - echo $INSTANCE_ID > $INSTANCE_ID_FILE - needs: - - job: build for linux amd64 - artifacts: - paths: - - $INSTANCE_ID_FILE - needs: - - job: build for linux amd64 - -# scp featurebase binary, config and service file, and configure featurebase -initiate featurebase linux amd64 in node: - stage: integration - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - IP: "" - PROFILE: "default" - AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY - INSTANCE_ID: "" - before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - - aws configure set aws_profile $PROFILE - - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - - chmod 400 gitlab-featurebase-dev.pem - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - - eval `ssh-agent -s` - - mkdir -p ~/.ssh - - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - - - chmod 700 /root/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - INSTANCE_ID=$(cat $INSTANCE_ID_FILE) - script: - - for i in {0..24}; do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE); if [ -n "$IP" ]; then break; fi; sleep 5; done - - scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. - - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region us-east-2 - needs: - - job: build for linux amd64 - - job: deploy node for linux amd64 - -# terminate EC2 instance -terminate node for linux amd64: - stage: integration - image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variables: - INSTANCE_ID: "" - before_script: - - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - - aws configure set region "us-east-2" - - INSTANCE_ID=$(cat $INSTANCE_ID_FILE) - script: - - aws ec2 terminate-instances --instance-ids $INSTANCE_ID - needs: - - job: initiate featurebase linux amd64 in node - when: always + - ./qa/scripts/deployNode.sh diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh new file mode 100755 index 000000000..42af61d20 --- /dev/null +++ b/qa/scripts/deployNode.sh @@ -0,0 +1,96 @@ +#! /bin/bash + +# To run script: ./deployNode.sh AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SSH_PRIVATE_KEY + +function configure_env() { + aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + aws configure set region $REGION + aws configure set aws_profile $PROFILE + echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem + chmod 400 gitlab-featurebase-dev.pem + + # set up ssh permissions + 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + eval `ssh-agent -s` + mkdir -p ~/.ssh + echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - + chmod 700 /root/.ssh + + # install jq + apt update && apt -y install jq +} + +function deploy_node() { + # get AMI, security group and subnet ID + AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) + SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) + SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) + + # launch EC2 instance and get instance ID + aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json + INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') +} + +function initialize_featurebase() { + # get IP for node + for i in {0..24} + do + IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) + if [ -n "$IP" ]; then + break + fi + + sleep 5 + done + } + + # copy featurebase binary and files to ec2 instance + scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. + + # execute script to configure featurebase on the EC2 node + aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region $REGION +} + +function terminate_node() { + aws ec2 terminate-instances --instance-ids $INSTANCE_ID +} + +# Pass variables to shell script +AWS_ACCESS_KEY_ID=$1 +shift + +AWS_SECRET_ACCESS_KEY=$1 +shift + +AWS_SSH_PRIVATE_KEY=$1 +shift + +# set some variables +PROFILE="default" +INSTANCE="t3a.large" +REGION="us-east-2" + +# set up environment variables for aws, ssh +# and install needed packages +configure_env +if [ $? > 0 ]; then + exit 1 +fi + +# get AMI, security group and subnet for EC2 instance, +# launch instance, save instance Id and run cloud-init to set up node env +deploy_node +if [ $? > 0 ]; then + exit 1 +fi + +# Get IP for instance, scp featurebase binary, config and service files; +# set up featurebase config in node +initialize_featurebase +if [ $? > 0 ]; then + terminate_node + exit 1 +else + terminate_node +fi \ No newline at end of file From 97841608e01dee30b8d17fe9da1e32f277ca0835 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 15:26:42 -0600 Subject: [PATCH 197/222] switched stages to make testing faster --- .gitlab/.gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 656e4e4d5..cf422fab5 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -14,11 +14,10 @@ variables: GOVERSION: "1.16.9" stages: - - integration - lint - test - build - + - integration #before_script: #- echo "before_script" @@ -115,7 +114,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: integration + stage: lint image: golang:1.16.10 script: - rm -r lattice @@ -199,7 +198,7 @@ build container fb: # deploy EC2 instance and save instance ID to a text file deploy node for linux amd64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest script: - ./qa/scripts/deployNode.sh From aac3b0768c6f2c1eaf0fc4e7dcd81b5c2c4e57be Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 15:32:22 -0600 Subject: [PATCH 198/222] fixed syntax error --- qa/scripts/deployNode.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 42af61d20..4ffc4720c 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -43,7 +43,6 @@ function initialize_featurebase() { sleep 5 done - } # copy featurebase binary and files to ec2 instance scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. From 53766f47ca6dfdd97aa5f1eb5e8997d12c981a19 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 15:38:09 -0600 Subject: [PATCH 199/222] fixed jobs dependencies and added args --- .gitlab/.gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index cf422fab5..eb4f6b374 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -201,4 +201,6 @@ deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest script: - - ./qa/scripts/deployNode.sh + - ./qa/scripts/deployNode.sh $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $AWS_SSH_PRIVATE_KEY + needs: + - job: build for linux amd64 \ No newline at end of file From dbd1e611d0a7bf901bd2d6f2cdea6df982133d86 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 15:42:12 -0600 Subject: [PATCH 200/222] changed jobs order --- .gitlab/.gitlab-ci.yml | 4 +--- qa/scripts/deployNode.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index eb4f6b374..6390da8ad 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -114,7 +114,7 @@ upload to sonarcloud: - job: run jest tests build for linux amd64: - stage: lint + stage: build image: golang:1.16.10 script: - rm -r lattice @@ -202,5 +202,3 @@ deploy node for linux amd64: image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest script: - ./qa/scripts/deployNode.sh $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $AWS_SSH_PRIVATE_KEY - needs: - - job: build for linux amd64 \ No newline at end of file diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 4ffc4720c..d790d3e12 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#!/bin/bash # To run script: ./deployNode.sh AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SSH_PRIVATE_KEY From a6bc10dec9fb4d2c1338b587701250d47483dad3 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 15:50:43 -0600 Subject: [PATCH 201/222] removed single quotes --- qa/scripts/deployNode.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index d790d3e12..6455c8ea1 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -11,7 +11,7 @@ function configure_env() { chmod 400 gitlab-featurebase-dev.pem # set up ssh permissions - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y ) eval `ssh-agent -s` mkdir -p ~/.ssh echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - From cceb173d5d08daf4a75c60d0a6c27ed8e7843b24 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 15:55:36 -0600 Subject: [PATCH 202/222] added before script commands --- .gitlab/.gitlab-ci.yml | 3 +++ qa/scripts/deployNode.sh | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6390da8ad..b72fc4d35 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,5 +200,8 @@ build container fb: deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + before_script: + - apt update && apt -y install jq + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' script: - ./qa/scripts/deployNode.sh $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $AWS_SSH_PRIVATE_KEY diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 6455c8ea1..7ba33a516 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -11,14 +11,10 @@ function configure_env() { chmod 400 gitlab-featurebase-dev.pem # set up ssh permissions - which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y ) eval `ssh-agent -s` mkdir -p ~/.ssh - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - + echo $AWS_SSH_PRIVATE_KEY | ssh-add - chmod 700 /root/.ssh - - # install jq - apt update && apt -y install jq } function deploy_node() { From 53d7d6b91c2bfbec713a42186f59b3f5b5d91b3b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 16:04:29 -0600 Subject: [PATCH 203/222] fix for ssh-add error --- .gitlab/.gitlab-ci.yml | 3 --- qa/scripts/deployNode.sh | 7 ++++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index b72fc4d35..6390da8ad 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,8 +200,5 @@ build container fb: deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - before_script: - - apt update && apt -y install jq - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' script: - ./qa/scripts/deployNode.sh $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $AWS_SSH_PRIVATE_KEY diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 7ba33a516..b2ad98313 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -11,10 +11,15 @@ function configure_env() { chmod 400 gitlab-featurebase-dev.pem # set up ssh permissions + which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y ) eval `ssh-agent -s` mkdir -p ~/.ssh - echo $AWS_SSH_PRIVATE_KEY | ssh-add - + echo "${AWS_SSH_PRIVATE_KEY}" | ssh-add - chmod 700 /root/.ssh + + # install jq + apt update && apt -y install jq + } function deploy_node() { From 184b93555093444cb3eeb40c306d166dd7aef9fb Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 16:07:22 -0600 Subject: [PATCH 204/222] added print statement --- qa/scripts/deployNode.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index b2ad98313..5c0694aba 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -14,7 +14,8 @@ function configure_env() { which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y ) eval `ssh-agent -s` mkdir -p ~/.ssh - echo "${AWS_SSH_PRIVATE_KEY}" | ssh-add - + echo "$AWS_SSH_PRIVATE_KEY" + echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - chmod 700 /root/.ssh # install jq From 8a77da0db9162de8f07143fdf7a3cbc075bc0d71 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 16:14:21 -0600 Subject: [PATCH 205/222] removed ssh keys from shell script --- .gitlab/.gitlab-ci.yml | 10 ++++++++++ qa/scripts/deployNode.sh | 14 -------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6390da8ad..37506c709 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,5 +200,15 @@ build container fb: deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + before_script: + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem + - chmod 400 gitlab-featurebase-dev.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval `ssh-agent -s` + - mkdir -p ~/.ssh && + - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq script: - ./qa/scripts/deployNode.sh $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $AWS_SSH_PRIVATE_KEY diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 5c0694aba..4d89805a2 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -7,20 +7,6 @@ function configure_env() { aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY aws configure set region $REGION aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - chmod 400 gitlab-featurebase-dev.pem - - # set up ssh permissions - which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y ) - eval `ssh-agent -s` - mkdir -p ~/.ssh - echo "$AWS_SSH_PRIVATE_KEY" - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - - chmod 700 /root/.ssh - - # install jq - apt update && apt -y install jq - } function deploy_node() { From d781dfb29b4d9754182748f90c289c053b1d825f Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 16:18:14 -0600 Subject: [PATCH 206/222] remove error handling to debug --- qa/scripts/deployNode.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 4d89805a2..a850e8403 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -53,6 +53,8 @@ shift AWS_SSH_PRIVATE_KEY=$1 shift +echo $AWS_ACCESS_KEY_ID, $AWS_ACCESS_KEY_ID, $AWS_SSH_PRIVATE_KEY + # set some variables PROFILE="default" INSTANCE="t3a.large" @@ -61,23 +63,23 @@ REGION="us-east-2" # set up environment variables for aws, ssh # and install needed packages configure_env -if [ $? > 0 ]; then - exit 1 -fi +# if [ $? > 0 ]; then +# exit 1 +# fi # get AMI, security group and subnet for EC2 instance, # launch instance, save instance Id and run cloud-init to set up node env deploy_node -if [ $? > 0 ]; then - exit 1 -fi +# if [ $? > 0 ]; then +# exit 1 +# fi # Get IP for instance, scp featurebase binary, config and service files; # set up featurebase config in node initialize_featurebase -if [ $? > 0 ]; then - terminate_node - exit 1 -else - terminate_node -fi \ No newline at end of file +# if [ $? > 0 ]; then +# terminate_node +# exit 1 +# else +terminate_node +# fi \ No newline at end of file From 0030e759da0963274e99c6c778eeb7bce1ea6b76 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 16:24:56 -0600 Subject: [PATCH 207/222] removed aws config from shell script --- .gitlab/.gitlab-ci.yml | 12 ++++++++++-- qa/scripts/deployNode.sh | 28 +++------------------------- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 37506c709..f568209b8 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,15 +200,23 @@ build container fb: deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variable: + PROFILE: "default" + AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: + - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - chmod 400 gitlab-featurebase-dev.pem - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` - - mkdir -p ~/.ssh && + - mkdir -p ~/.ssh - echo "$AWS_SSH_PRIVATE_KEY" | ssh-add - - chmod 700 /root/.ssh - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - apt update && apt -y install jq script: - - ./qa/scripts/deployNode.sh $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY $AWS_SSH_PRIVATE_KEY + - ./qa/scripts/deployNode.sh $PROFILE diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index a850e8403..3f3d4823b 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -1,13 +1,6 @@ #!/bin/bash -# To run script: ./deployNode.sh AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SSH_PRIVATE_KEY - -function configure_env() { - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region $REGION - aws configure set aws_profile $PROFILE -} +# To run script: ./deployNode.sh $PROFILE function deploy_node() { # get AMI, security group and subnet ID @@ -43,30 +36,15 @@ function terminate_node() { aws ec2 terminate-instances --instance-ids $INSTANCE_ID } -# Pass variables to shell script -AWS_ACCESS_KEY_ID=$1 -shift +# Pass variables to shell script -AWS_SECRET_ACCESS_KEY=$1 -shift - -AWS_SSH_PRIVATE_KEY=$1 +PROFILE=$1 shift -echo $AWS_ACCESS_KEY_ID, $AWS_ACCESS_KEY_ID, $AWS_SSH_PRIVATE_KEY - # set some variables -PROFILE="default" INSTANCE="t3a.large" REGION="us-east-2" -# set up environment variables for aws, ssh -# and install needed packages -configure_env -# if [ $? > 0 ]; then -# exit 1 -# fi - # get AMI, security group and subnet for EC2 instance, # launch instance, save instance Id and run cloud-init to set up node env deploy_node From 2de7d2c885e87679b091e63b4ace72ccd17852c4 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 16:26:09 -0600 Subject: [PATCH 208/222] fixed yaml syntax error --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f568209b8..3fe3e86aa 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -200,7 +200,7 @@ build container fb: deploy node for linux amd64: stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest - variable: + variables: PROFILE: "default" AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY before_script: From f03720f0a3fb234ffa895014a2c0ea978f7f2da4 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 16:33:32 -0600 Subject: [PATCH 209/222] error handling --- qa/scripts/deployNode.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 3f3d4823b..3511c78b7 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -48,16 +48,16 @@ REGION="us-east-2" # get AMI, security group and subnet for EC2 instance, # launch instance, save instance Id and run cloud-init to set up node env deploy_node -# if [ $? > 0 ]; then -# exit 1 -# fi +if [ $? > 0 ]; then + exit 1 +fi # Get IP for instance, scp featurebase binary, config and service files; # set up featurebase config in node initialize_featurebase -# if [ $? > 0 ]; then -# terminate_node -# exit 1 -# else -terminate_node -# fi \ No newline at end of file +if [ $? > 0 ]; then + terminate_node + exit 1 +else + terminate_node +fi \ No newline at end of file From 88c89749afbf2ce7ac8cca0bcc596c7b82742123 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 16:37:11 -0600 Subject: [PATCH 210/222] updated job's stage --- .gitlab/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 3fe3e86aa..0568256ed 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -198,7 +198,7 @@ build container fb: # deploy EC2 instance and save instance ID to a text file deploy node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: PROFILE: "default" From 612e11887f8d58da308ac18152a23c93e15b4ecb Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 16:41:35 -0600 Subject: [PATCH 211/222] added dependency --- .gitlab/.gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 0568256ed..18e7d76a2 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -220,3 +220,5 @@ deploy node for linux amd64: - apt update && apt -y install jq script: - ./qa/scripts/deployNode.sh $PROFILE + needs: + - job: build for linux amd64 From d13eef4c4b4adf6d79bd30c791ffa742901fd39b Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 17:09:35 -0600 Subject: [PATCH 212/222] echo errors --- .gitlab/.gitlab-ci.yml | 3 +-- qa/scripts/deployNode.sh | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 18e7d76a2..f2dbeee30 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -198,7 +198,7 @@ build container fb: # deploy EC2 instance and save instance ID to a text file deploy node for linux amd64: - stage: integration + stage: lint image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: PROFILE: "default" @@ -209,7 +209,6 @@ deploy node for linux amd64: - aws configure set region "us-east-2" - aws configure set aws_profile $PROFILE - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - - echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem - chmod 400 gitlab-featurebase-dev.pem - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval `ssh-agent -s` diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 3511c78b7..816f87a48 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -49,6 +49,7 @@ REGION="us-east-2" # launch instance, save instance Id and run cloud-init to set up node env deploy_node if [ $? > 0 ]; then + echo "Error: " >&2 exit 1 fi @@ -57,7 +58,9 @@ fi initialize_featurebase if [ $? > 0 ]; then terminate_node + echo "Error: " >&2 exit 1 else terminate_node + exit 0 fi \ No newline at end of file From b243f4b8a3c623a67d9df6121bfdca29467f00f7 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 17:10:13 -0600 Subject: [PATCH 213/222] fix yaml error --- .gitlab/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f2dbeee30..19c984175 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -219,5 +219,5 @@ deploy node for linux amd64: - apt update && apt -y install jq script: - ./qa/scripts/deployNode.sh $PROFILE - needs: - - job: build for linux amd64 + # needs: + # - job: build for linux amd64 From afb96f4a9bc727e63f27e1072e51f05ab91bf63d Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 17:17:13 -0600 Subject: [PATCH 214/222] pass error back to gitlab runner --- qa/scripts/deployNode.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 816f87a48..60a007d93 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -58,7 +58,7 @@ fi initialize_featurebase if [ $? > 0 ]; then terminate_node - echo "Error: " >&2 + echo "Error: " 1>&2 exit 1 else terminate_node From 42b6c2b351bc50a5a46c696e52d08a2d3d7b5443 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 17:25:10 -0600 Subject: [PATCH 215/222] updated syntax --- qa/scripts/deployNode.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 60a007d93..ac3364472 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -11,6 +11,7 @@ function deploy_node() { # launch EC2 instance and get instance ID aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') + echo "Instance ID: " $INSTANCE_ID } function initialize_featurebase() { @@ -24,6 +25,7 @@ function initialize_featurebase() { sleep 5 done + echo "IP: " $IP # copy featurebase binary and files to ec2 instance scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. @@ -49,7 +51,7 @@ REGION="us-east-2" # launch instance, save instance Id and run cloud-init to set up node env deploy_node if [ $? > 0 ]; then - echo "Error: " >&2 + echo "Error: " 1>&2 exit 1 fi @@ -57,8 +59,8 @@ fi # set up featurebase config in node initialize_featurebase if [ $? > 0 ]; then - terminate_node echo "Error: " 1>&2 + terminate_node exit 1 else terminate_node From 001678386f26ca2b40fcc760a97a6fc49a2362c5 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 17:27:57 -0600 Subject: [PATCH 216/222] define ip var --- qa/scripts/deployNode.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index ac3364472..9bc064beb 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -16,6 +16,7 @@ function deploy_node() { function initialize_featurebase() { # get IP for node + IP="" for i in {0..24} do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) From 6172f98eaba4d2025dd6a0489b10d1d2bbae0b1c Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 18:32:09 -0600 Subject: [PATCH 217/222] test failed condition --- qa/scripts/deployNode.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 9bc064beb..a3b20f6b3 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -10,13 +10,12 @@ function deploy_node() { # launch EC2 instance and get instance ID aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json - INSTANCE_ID=$(jq '.Instances | .[] |.InstanceId' config.json | tr -d '"') + INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"') echo "Instance ID: " $INSTANCE_ID } function initialize_featurebase() { # get IP for node - IP="" for i in {0..24} do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) @@ -26,7 +25,8 @@ function initialize_featurebase() { sleep 5 done - echo "IP: " $IP + + sleep 60 # to allow enough time for node to be ready for use # copy featurebase binary and files to ec2 instance scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. @@ -51,16 +51,14 @@ REGION="us-east-2" # get AMI, security group and subnet for EC2 instance, # launch instance, save instance Id and run cloud-init to set up node env deploy_node -if [ $? > 0 ]; then - echo "Error: " 1>&2 +if [[ $? > 0 ]]; then exit 1 fi # Get IP for instance, scp featurebase binary, config and service files; # set up featurebase config in node initialize_featurebase -if [ $? > 0 ]; then - echo "Error: " 1>&2 +if [[ $? > 0 ]]; then terminate_node exit 1 else From 0e2d71c3030fe31132b339f4162d08e31fc36054 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 18:48:06 -0600 Subject: [PATCH 218/222] added error handling --- qa/scripts/deployNode.sh | 56 ++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index a3b20f6b3..2074d5400 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -5,13 +5,32 @@ function deploy_node() { # get AMI, security group and subnet ID AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) + if [[ $? > 0 ]]; then + echo "aws session manager find ami failed" + exit 1 + fi + SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) + if [[ $? > 0 ]]; then + echo "aws session manager find security group failed" + exit 1 + fi + SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) + if [[ $? > 0 ]]; then + echo "aws session manager find subnet ID failed" + exit 1 + fi # launch EC2 instance and get instance ID aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json + if [[ $? > 0 ]]; then + echo "Launching EC2 instance failed" + exit 1 + fi + INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"') - echo "Instance ID: " $INSTANCE_ID + echo "Launched EC2 instance with Instance ID: " $INSTANCE_ID } function initialize_featurebase() { @@ -19,10 +38,17 @@ function initialize_featurebase() { for i in {0..24} do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) - if [ -n "$IP" ]; then + if [ -n "$IP" ]; then + echo "Public IP for EC2 instance: " $IP break fi + if [[ $? > 0 ]]; then + echo "aws cli describe-instances command failed" + terminate_node + exit 1 + fi + sleep 5 done @@ -30,17 +56,26 @@ function initialize_featurebase() { # copy featurebase binary and files to ec2 instance scp -o StrictHostKeyChecking=no -i gitlab-featurebase-dev.pem featurebase_linux_amd64 ./qa/scripts/featurebase.conf ./qa/scripts/featurebase.service ec2-user@$IP:. + if [[ $? > 0 ]]; then + echo "scp of featurebase binary, service and config files to EC2 instance failed" + terminate_node + exit 1 + fi # execute script to configure featurebase on the EC2 node aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region $REGION + if [[ $? > 0 ]]; then + echo "aws cli session manager command failed" + terminate_node + exit 1 + fi } function terminate_node() { - aws ec2 terminate-instances --instance-ids $INSTANCE_ID + aws ec2 terminate-instances --instance-ids $INSTANCE_ID --profile $PROFILE } # Pass variables to shell script - PROFILE=$1 shift @@ -51,17 +86,10 @@ REGION="us-east-2" # get AMI, security group and subnet for EC2 instance, # launch instance, save instance Id and run cloud-init to set up node env deploy_node -if [[ $? > 0 ]]; then - exit 1 -fi # Get IP for instance, scp featurebase binary, config and service files; # set up featurebase config in node initialize_featurebase -if [[ $? > 0 ]]; then - terminate_node - exit 1 -else - terminate_node - exit 0 -fi \ No newline at end of file + +# terminate node +terminate_node From 02b0f4ffd66bf3f7da3d68f0d41775f91e7d02d9 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 18:50:47 -0600 Subject: [PATCH 219/222] test standard condition --- .gitlab/.gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 19c984175..f75bf268f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -198,7 +198,7 @@ build container fb: # deploy EC2 instance and save instance ID to a text file deploy node for linux amd64: - stage: lint + stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest variables: PROFILE: "default" @@ -219,5 +219,5 @@ deploy node for linux amd64: - apt update && apt -y install jq script: - ./qa/scripts/deployNode.sh $PROFILE - # needs: - # - job: build for linux amd64 + needs: + - job: build for linux amd64 From 65c4fc9cdb88b28a5a4f4008a9863fae7d87f5d0 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Tue, 23 Nov 2021 20:28:14 -0600 Subject: [PATCH 220/222] updated error messages --- .gitlab/.gitlab-ci.yml | 2 +- qa/scripts/deployNode.sh | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f75bf268f..f0a523e1a 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -196,7 +196,7 @@ build container fb: - docker push $tag - echo Created docker featurebase image with tag "$tag" -# deploy EC2 instance and save instance ID to a text file +# deploy EC2 instance, configure and run featurebase deploy node for linux amd64: stage: integration image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest diff --git a/qa/scripts/deployNode.sh b/qa/scripts/deployNode.sh index 2074d5400..b928a31e7 100755 --- a/qa/scripts/deployNode.sh +++ b/qa/scripts/deployNode.sh @@ -6,31 +6,31 @@ function deploy_node() { # get AMI, security group and subnet ID AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE) if [[ $? > 0 ]]; then - echo "aws session manager find ami failed" + echo "aws session manager failed to find AMI" exit 1 fi SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE) if [[ $? > 0 ]]; then - echo "aws session manager find security group failed" + echo "aws session manager failed to find security group" exit 1 fi SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE) if [[ $? > 0 ]]; then - echo "aws session manager find subnet ID failed" + echo "aws session manager failed to find subnet ID" exit 1 fi # launch EC2 instance and get instance ID aws ec2 run-instances --image-id $AMI --instance-type $INSTANCE --security-group-ids $SECURITY_GROUP --subnet-id $SUBNET_ID --key-name gitlab-featurebase-dev --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=linux-amd64-node}]' --profile $PROFILE --user-data file://./qa/scripts/cloud-init.sh --iam-instance-profile Name=featurebase-dev-ssm > config.json if [[ $? > 0 ]]; then - echo "Launching EC2 instance failed" + echo "aws run-instances failed to launch a new EC2 instance" exit 1 fi INSTANCE_ID=$(jq '.Instances | .[0] |.InstanceId' config.json | tr -d '"') - echo "Launched EC2 instance with Instance ID: " $INSTANCE_ID + echo "aws run-instances succeeded in launching a new EC2 instance with instance ID: " $INSTANCE_ID } function initialize_featurebase() { @@ -39,12 +39,12 @@ function initialize_featurebase() { do IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --filters 'Name=instance-state-name, Values=running' --query 'Reservations[*].Instances[*].PublicIpAddress' --output text --profile $PROFILE) if [ -n "$IP" ]; then - echo "Public IP for EC2 instance: " $IP + echo "Public IP for EC2 instance found: " $IP break fi if [[ $? > 0 ]]; then - echo "aws cli describe-instances command failed" + echo "aws cli describe-instances command failed to find public IP" terminate_node exit 1 fi @@ -65,7 +65,7 @@ function initialize_featurebase() { # execute script to configure featurebase on the EC2 node aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID --cli-input-json file://./qa/scripts/configureFeatureBase.json --profile $PROFILE --region $REGION if [[ $? > 0 ]]; then - echo "aws cli session manager command failed" + echo "aws cli session manager send-command failed" terminate_node exit 1 fi From 8b7d429417d2e430127941d809ce602e0efad64d Mon Sep 17 00:00:00 2001 From: tgruben Date: Mon, 29 Nov 2021 09:18:44 -0600 Subject: [PATCH 221/222] Update README.md --- rbf/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rbf/README.md b/rbf/README.md index 4f6efa072..cbc5b74a7 100644 --- a/rbf/README.md +++ b/rbf/README.md @@ -93,12 +93,12 @@ The leaf page contains a series of cells with the header of: [8] highbits [4] flag [4] child count - [*] array or RLE data + [*] array or RLE data or Handle (a pageno) to Bitmap Data -### Bitmap page +### Bitmap Data page -The data for the bitmap page takes up the entire 8KB. +The data for the bitmap data page takes up the entire 8KB. ## Proof of Concept Notes From 1df90af26e98bbcd35383257af1e1aed4a9899f6 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 2 Dec 2021 15:57:52 -0600 Subject: [PATCH 222/222] free bitmap pages on deallocate --- client/client_it_test.go | 1 - rbf/README.md | 6 +-- rbf/tx.go | 8 +++- rbf/tx_test.go | 99 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 5 deletions(-) diff --git a/client/client_it_test.go b/client/client_it_test.go index 1b5ca2884..b9137bee9 100644 --- a/client/client_it_test.go +++ b/client/client_it_test.go @@ -592,7 +592,6 @@ func TestClientAgainstCluster(t *testing.T) { target := []uint64{100} require.Equalf(t, target, resp.Result().Row().Columns, "Row Result Columns") }) - t.Run("StoreQuery", func(t *testing.T) { schema := NewSchema() testIndexStore := schema.Index("test-index-store") diff --git a/rbf/README.md b/rbf/README.md index 4f6efa072..cbc5b74a7 100644 --- a/rbf/README.md +++ b/rbf/README.md @@ -93,12 +93,12 @@ The leaf page contains a series of cells with the header of: [8] highbits [4] flag [4] child count - [*] array or RLE data + [*] array or RLE data or Handle (a pageno) to Bitmap Data -### Bitmap page +### Bitmap Data page -The data for the bitmap page takes up the entire 8KB. +The data for the bitmap data page takes up the entire 8KB. ## Proof of Concept Notes diff --git a/rbf/tx.go b/rbf/tx.go index 812d35738..52cc49366 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -333,7 +333,6 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error { if !strings.HasPrefix(name.(string), prefix) { continue } - // Deallocate all pages in the tree. if err := tx.deallocateTree(pgno.(uint32)); err != nil { return err @@ -1065,6 +1064,13 @@ func (tx *Tx) deallocateTree(pgno uint32) error { return tx.freePgno(pgno) case PageTypeLeaf: + for i, n := 0, readCellN(page); i < n; i++ { + if cell := readLeafCell(page, i); cell.Type == ContainerTypeBitmapPtr { + if err := tx.freePgno(toPgno(cell.Data)); err != nil { + return err + } + } + } return tx.freePgno(pgno) default: return fmt.Errorf("rbf.Tx.deallocateTree(): invalid page type: pgno=%d type=%d", pgno, typ) diff --git a/rbf/tx_test.go b/rbf/tx_test.go index e4b68f4b6..8a354a916 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -684,3 +684,102 @@ func TestTx_CreateBitmap(t *testing.T) { } }) } + +func TestTx_DeleteBitmapsWithPrefix(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + prefix := "abc" + bitmapSize := 10000 + // var err error + // create a interleaved set up array and bitmap containers + + bits := make([]uint64, bitmapSize) + x := uint64(1) + for i := 0; i < len(bits); i++ { + bits[i] = x + x = x + 2 + } + ifError := func(err error) { + if err != nil { + t.Fatal(err) + } + } + checkInfos := func() { + tx := MustBegin(t, db, false) + defer tx.Rollback() + infos, err := tx.PageInfos() + ifError(err) + for pgno, info := range infos { + switch info := info.(type) { + case *rbf.MetaPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "meta") + fmt.Printf("pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo) + + case *rbf.RootRecordPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "rootrec") + fmt.Printf("next=%d\n", info.Next) + + case *rbf.LeafPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "leaf") + fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN) + + case *rbf.BranchPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "branch") + fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN) + + case *rbf.BitmapPageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "bitmap") + fmt.Printf("-\n") + + case *rbf.FreePageInfo: + fmt.Printf("%-8d ", pgno) + fmt.Printf("%-10s ", "free") + fmt.Printf("-\n") + + default: + t.Fatal(fmt.Sprintf("unexpected page info type %T", info)) + } + } + + } + populate := func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + for i := uint64(0); i < 16; i++ { + bm := roaring.NewBitmap() + if i%3 == 0 { + bm.Put(i, roaring.NewContainerBitmap(6144, bits)) + if _, err := tx.AddRoaring(prefix, bm); err != nil { + panic(err) + } + } else { + bm.Put(i, roaring.NewContainerArray([]uint16{2, 4, 5, 7})) + if _, err := tx.AddRoaring(prefix, bm); err != nil { + panic(err) + } + + } + } + ifError(tx.Commit()) + } + + checkInfos() + populate() + checkInfos() + ifError(db.Check()) + + tx := MustBegin(t, db, true) + tx.DeleteBitmapsWithPrefix(prefix) + ifError(tx.Commit()) + ifError(db.Check()) + checkInfos() + populate() + ifError(db.Check()) + checkInfos() + +}