scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 19:37:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into protect-endpoints

Browse source
This commit is contained in:
reesporte 2022-01-04 16:18:18 -06:00
commit 8d6490329b
91 changed files with 2697 additions and 690 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
.artifactory/pipelines.yml
View file

@ -1,38 +0,0 @@
resources:
- name: featurebaseRepo
type: GitRepo
configuration:
# SCM integration where the repository is located
gitProvider: github_molecula_featurebase
# Repository path, including org name/repo name
path: molecula/featurebase
branches:
# Specifies which branches will trigger dependent steps
include: cicd
- name: featurebaseBuildInfo
type: BuildInfo
configuration:
sourceArtifactory: Molecula_Artifactory
buildName: featurebase_build
buildNumber: 4
pipelines:
- name: ScanGoCode
steps:
- name: scan
type: XrayScan
configuration:
failOnScan: false
inputResources:
- name: featurebaseBuildInfo
trigger: true
execution:
onStart:
- echo "Preparing for work..."
- echo "Prepping build environment"
onSuccess:
- echo "Job well done!"
onFailure:
- echo "uh oh, something went wrong"
onComplete:
- echo "Cleaning up some stuff"

602
.circleci/config.yml
View file

@ -1,305 +1,305 @@
version: 2.1
# version: 2.1
executors:
golang:
parameters:
version:
type: string
default: "1.15.8"
resource_class:
type: string
default: medium
docker:
- image: circleci/golang:<< parameters.version >>
resource_class: << parameters.resource_class >>
working_directory: /go/src/github.com/molecula/featurebase
# executors:
# golang:
# parameters:
# version:
# type: string
# default: "1.15.8"
# resource_class:
# type: string
# default: medium
# docker:
# - image: circleci/golang:<< parameters.version >>
# resource_class: << parameters.resource_class >>
# working_directory: /go/src/github.com/molecula/featurebase
commands:
add-github-auth:
steps:
- run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
- run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:"
restore-mod-cache:
steps:
- restore_cache:
key: mod-cache-{{ checksum "go.sum" }}
save-mod-cache:
steps:
- save_cache:
key: mod-cache-{{ checksum "go.sum" }}
paths:
- /go/pkg/mod/
checkout-plus:
steps:
- add-github-auth
- checkout
- restore-mod-cache
skip-if-root-unchanged:
description: "skips the parent job if the PR includes no changes to featurebase"
steps:
- run: |
ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true
echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES"
if [ -z "$ROOT_CHANGED_FILES" ] ; then
echo "halting step"
circleci step halt
fi
skip-if-lattice-unchanged:
description: "skips the parent job if the PR includes no changes to lattice"
steps:
- run: |
LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true
echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES"
if [ -z "$LATTICE_CHANGED_FILES" ] ; then
echo "halting step"
circleci step halt
fi
# commands:
# add-github-auth:
# steps:
# - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
# - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:"
# restore-mod-cache:
# steps:
# - restore_cache:
# key: mod-cache-{{ checksum "go.sum" }}
# save-mod-cache:
# steps:
# - save_cache:
# key: mod-cache-{{ checksum "go.sum" }}
# paths:
# - /go/pkg/mod/
# checkout-plus:
# steps:
# - add-github-auth
# - checkout
# - restore-mod-cache
# skip-if-root-unchanged:
# description: "skips the parent job if the PR includes no changes to featurebase"
# steps:
# - run: |
# ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true
# echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES"
# if [ -z "$ROOT_CHANGED_FILES" ] ; then
# echo "halting step"
# circleci step halt
# fi
# skip-if-lattice-unchanged:
# description: "skips the parent job if the PR includes no changes to lattice"
# steps:
# - run: |
# LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true
# echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES"
# if [ -z "$LATTICE_CHANGED_FILES" ] ; then
# echo "halting step"
# circleci step halt
# fi
jobs:
setup:
executor:
name: golang
steps:
- checkout-plus
- run: go mod download
- save-mod-cache
linter:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0
- run: make golangci-lint
go-mod-tidy:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- run: go mod tidy
- run: git diff --exit-code -- go.mod go.sum
check-changelog-label:
executor:
name: golang
steps:
- run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request
- run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e
test-build-arm:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- run: make build GOOS=linux GOARCH=arm GOARM=5
- run: make build GOOS=linux GOARCH=arm GOARM=6
- run: make build GOOS=linux GOARCH=arm GOARM=7
- run: make build GOOS=linux GOARCH=arm64
test:
parameters:
resource_class:
type: string
default: medium
golang_version:
type: string
default: "1.15.8"
shard_width:
type: string
default: "20"
test_make_target:
type: string
default: "test"
test_flags:
type: string
default: ""
goarch:
type: string
default: amd64
executor:
name: golang
version: << parameters.golang_version >>
resource_class: << parameters.resource_class >>
environment:
TMPDIR: /mnt/ramdisk
steps:
- checkout-plus
- skip-if-root-unchanged
- run: sudo apt-get update --allow-releaseinfo-change -y
- run: sudo apt-get install lsof
- run:
command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
no_output_timeout: 30m
test-external-lookup:
docker:
- image: circleci/golang:1.15.8
- image: circleci/postgres:13.2-ram
environment:
POSTGRES_PASSWORD=password
steps:
- checkout-plus
- skip-if-root-unchanged
- run: sudo apt-get update --allow-releaseinfo-change -y
- run: sudo apt-get install postgresql-client
- run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1)
- run:
command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
no_output_timeout: 30m
cluster-tests:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- setup_remote_docker
- run: make clustertests-build
release:
executor:
name: golang
steps:
- checkout-plus
- attach_workspace:
at: .
- setup_remote_docker:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker-release
- store_artifacts:
path: build
- persist_to_workspace:
root: .
paths: build
publish_release:
executor:
name: golang
steps:
- attach_workspace:
at: .
- run: go get github.com/tcnksm/ghr
- run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/
docker-build:
executor:
name: golang
steps:
- checkout-plus
- setup_remote_docker:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker GO_VERSION=1.15.8
- run: docker run featurebase:$(git describe --tags) help
dockerhub-upload-unstable:
executor:
name: golang
steps:
- checkout-plus
- setup_remote_docker:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker
- run: docker run featurebase:$(git describe --tags) help
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >>
dockerhub-upload-stable:
executor:
name: golang
steps:
- checkout-plus
- setup_remote_docker:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker
- run: docker run featurebase:$(git describe --tags) help
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >>
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest
# jobs:
# setup:
# executor:
# name: golang
# steps:
# - checkout-plus
# - run: go mod download
# - save-mod-cache
# linter:
# executor:
# name: golang
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0
# - run: make golangci-lint
# go-mod-tidy:
# executor:
# name: golang
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: go mod tidy
# - run: git diff --exit-code -- go.mod go.sum
# check-changelog-label:
# executor:
# name: golang
# steps:
# - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request
# - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e
# test-build-arm:
# executor:
# name: golang
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: make build GOOS=linux GOARCH=arm GOARM=5
# - run: make build GOOS=linux GOARCH=arm GOARM=6
# - run: make build GOOS=linux GOARCH=arm GOARM=7
# - run: make build GOOS=linux GOARCH=arm64
# test:
# parameters:
# resource_class:
# type: string
# default: medium
# golang_version:
# type: string
# default: "1.15.8"
# shard_width:
# type: string
# default: "20"
# test_make_target:
# type: string
# default: "test"
# test_flags:
# type: string
# default: ""
# goarch:
# type: string
# default: amd64
# executor:
# name: golang
# version: << parameters.golang_version >>
# resource_class: << parameters.resource_class >>
# environment:
# TMPDIR: /mnt/ramdisk
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: sudo apt-get update --allow-releaseinfo-change -y
# - run: sudo apt-get install lsof
# - run:
# command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
# no_output_timeout: 30m
# test-external-lookup:
# docker:
# - image: circleci/golang:1.15.8
# - image: circleci/postgres:13.2-ram
# environment:
# POSTGRES_PASSWORD=password
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: sudo apt-get update --allow-releaseinfo-change -y
# - run: sudo apt-get install postgresql-client
# - run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1)
# - run:
# command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
# no_output_timeout: 30m
# cluster-tests:
# executor:
# name: golang
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - setup_remote_docker
# - run: make clustertests
# release:
# executor:
# name: golang
# steps:
# - checkout-plus
# - attach_workspace:
# at: .
# - setup_remote_docker:
# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
# - run: make docker-release
# - store_artifacts:
# path: build
# - persist_to_workspace:
# root: .
# paths: build
# publish_release:
# executor:
# name: golang
# steps:
# - attach_workspace:
# at: .
# - run: go get github.com/tcnksm/ghr
# - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/
# docker-build:
# executor:
# name: golang
# steps:
# - checkout-plus
# - setup_remote_docker:
# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
# - run: make docker GO_VERSION=1.15.8
# - run: docker run featurebase:$(git describe --tags) help
# dockerhub-upload-unstable:
# executor:
# name: golang
# steps:
# - checkout-plus
# - setup_remote_docker:
# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
# - run: make docker
# - run: docker run featurebase:$(git describe --tags) help
# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >>
# dockerhub-upload-stable:
# executor:
# name: golang
# steps:
# - checkout-plus
# - setup_remote_docker:
# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
# - run: make docker
# - run: docker run featurebase:$(git describe --tags) help
# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >>
# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest
workflows:
build:
jobs:
- setup:
context: molecula
filters:
tags:
only: /^v.*/
- linter:
context: molecula
requires:
- setup
- go-mod-tidy:
context: molecula
requires:
- setup
- check-changelog-label:
context: molecula
requires:
- setup
- test-build-arm:
context: molecula
requires:
- setup
- test:
name: test-golang-<< matrix.golang_version >>
resource_class: large
context: molecula
requires:
- setup
matrix:
parameters:
golang_version: ["1.15.8", "1.16.10"]
- test:
name: << matrix.test_make_target >>
resource_class: xlarge
context: molecula
requires:
- setup
matrix:
parameters:
test_make_target: ["test-race"]
- test:
name: test-shardwidth-22
context: molecula
shard_width: "22"
resource_class: large
requires:
- setup
- test-external-lookup:
context: molecula
requires:
- setup
- cluster-tests:
context: molecula
requires:
- setup
- docker-build:
context: molecula
requires:
- setup
- release:
context: molecula
requires:
- setup
filters:
tags:
only: /^v.*/
- publish_release:
context: molecula
requires:
- release
filters:
tags:
only: /^v.*/
branches:
ignore: /.*/
- dockerhub-upload-unstable:
context: molecula
requires:
- setup
filters:
branches:
only: master
- dockerhub-upload-stable:
context: molecula
requires:
- setup
filters:
tags:
only: /^v.*/
branches:
ignore: /.*/
# workflows:
# build:
# jobs:
# - setup:
# context: molecula
# filters:
# tags:
# only: /^v.*/
# - linter:
# context: molecula
# requires:
# - setup
# - go-mod-tidy:
# context: molecula
# requires:
# - setup
# - check-changelog-label:
# context: molecula
# requires:
# - setup
# - test-build-arm:
# context: molecula
# requires:
# - setup
# - test:
# name: test-golang-<< matrix.golang_version >>
# resource_class: large
# context: molecula
# requires:
# - setup
# matrix:
# parameters:
# golang_version: ["1.15.8", "1.16.10"]
# - test:
# name: << matrix.test_make_target >>
# resource_class: xlarge
# context: molecula
# requires:
# - setup
# matrix:
# parameters:
# test_make_target: ["test-race"]
# - test:
# name: test-shardwidth-22
# context: molecula
# shard_width: "22"
# resource_class: large
# requires:
# - setup
# - test-external-lookup:
# context: molecula
# requires:
# - setup
# - cluster-tests:
# context: molecula
# requires:
# - setup
# - docker-build:
# context: molecula
# requires:
# - setup
# - release:
# context: molecula
# requires:
# - setup
# filters:
# tags:
# only: /^v.*/
# - publish_release:
# context: molecula
# requires:
# - release
# filters:
# tags:
# only: /^v.*/
# branches:
# ignore: /.*/
# - dockerhub-upload-unstable:
# context: molecula
# requires:
# - setup
# filters:
# branches:
# only: master
# - dockerhub-upload-stable:
# context: molecula
# requires:
# - setup
# filters:
# tags:
# only: /^v.*/
# branches:
# ignore: /.*/

5
.gitignore vendored
View file

@ -13,3 +13,8 @@ pilosa
*.dot
.idea/
.*.swp
.terraform/
*.tfstate
launch.json
.terraform.lock.hcl
__pycache__/

123
.gitlab/.gitlab-ci.yml
View file

@ -11,27 +11,23 @@ include:
paths:
- .go/pkg/mod/
variables:
GOVERSION: "1.16.9"
GOVERSION: "1.16.10"
stages:
- lint
- test
- build
- integration
#before_script:
#- echo "before_script"
#- git version
#- go env -w GOPRIVATE=github.com/molecula
#- mkdir -p .go
#- go version
#- go env -w GO111MODULE=on
- gauntlet
golangci-lint:
image: golangci/golangci-lint:v1.39.0
stage: lint
extends: .go-cache
allow_failure: false
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- echo "Checking for issues in new code"
- golangci-lint run -v
@ -41,6 +37,8 @@ build lattice:
image: node:14
variables:
CI: "false"
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- cd lattice
- yarn install
@ -59,6 +57,8 @@ run jest tests:
image: node:14
variables:
CI: "true"
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- echo "Testing lattice..."
- cd lattice
@ -70,8 +70,10 @@ run jest tests:
run go tests:
stage: test
image: golang:1.16.10
image: golang:$GOVERSION
extends: .go-cache
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- echo "Running featurebase unit tests..."
- PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
@ -84,6 +86,8 @@ run go tests future:
stage: test
image: golang:1.17.3
extends: .go-cache
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- echo "Running featurebase unit tests..."
- PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
@ -95,7 +99,9 @@ run go tests future:
run go tests with output:
stage: test
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- echo "Running featurebase unit tests to capture JSON output..."
- go test -json > test-report.out
@ -108,6 +114,8 @@ upload to sonarcloud:
image: sonarsource/sonar-scanner-cli:4.6
variables:
SONAR_TOKEN: $SONAR_TOKEN
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
needs:
@ -117,7 +125,9 @@ upload to sonarcloud:
build for linux amd64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- rm -r lattice
- tar -xvf lattice.tar.gz
@ -130,7 +140,9 @@ build for linux amd64:
build for linux arm64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- rm -r lattice
- tar -xvf lattice.tar.gz
@ -143,7 +155,9 @@ build for linux arm64:
build for darwin amd64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- rm -r lattice
- tar -xvf lattice.tar.gz
@ -156,7 +170,9 @@ build for darwin amd64:
build for darwin arm64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
script:
- rm -r lattice
- tar -xvf lattice.tar.gz
@ -169,7 +185,9 @@ build for darwin arm64:
package for linux amd64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
variables:
GOOS: "linux"
GOARCH: "amd64"
@ -190,6 +208,8 @@ build container fb:
- "build for linux amd64"
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
before_script:
- echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
script:
@ -199,27 +219,70 @@ build container fb:
- echo Created docker featurebase image with tag "$tag"
# deploy EC2 instance, configure and run featurebase
deploy node for linux amd64:
stage: integration
# diabling for now - will come back and refactor POK
# deploy node for linux amd64:
# stage: integration
# image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
# variables:
# PROFILE: "default"
# AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
# rules:
# - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
# before_script:
# - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
# - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
# - aws configure set region "us-east-2"
# - aws configure set aws_profile $PROFILE
# - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
# - chmod 400 gitlab-featurebase-ci.pem
# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
# - eval `ssh-agent -s`
# - mkdir -p ~/.ssh
# - echo "$AWS_FBCI_SSH_KEY" | ssh-add -
# - chmod 700 /root/.ssh
# - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
# - apt update && apt -y install jq
# script:
# - ./qa/scripts/deployNode.sh $PROFILE
# needs:
# - job: build for linux amd64
gauntlet:
stage: gauntlet
timeout: 4h
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
variables:
PROFILE: "default"
AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
before_script:
- aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
- aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- apt-get update && apt-get install terraform
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
- aws configure set region "us-east-2"
- aws configure set aws_profile $PROFILE
- echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem
- chmod 400 gitlab-featurebase-dev.pem
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
- chmod 400 gitlab-featurebase-ci.pem
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
- eval `ssh-agent -s`
- mkdir -p ~/.ssh
- echo "$AWS_SSH_PRIVATE_KEY" | ssh-add -
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
- chmod 700 /root/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- apt update && apt -y install jq
- apt update && apt -y install jq wget
- wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
- tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
- export PATH=$PATH:/usr/local/go/bin
script:
- ./qa/scripts/deployNode.sh $PROFILE
needs:
- job: build for linux amd64
- ./qa/scripts/setupSamsungGauntlet.sh
- ./qa/scripts/testSamsungGauntlet.sh
after_script:
- ./qa/scripts/teardownSamsungGauntlet.sh

8
Makefile
View file

@ -149,12 +149,10 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
clustertests: vendor
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) build
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
docker-compose -f $(DOCKER_COMPOSE) up -d pilosa1 pilosa2 pilosa3
docker-compose -f $(DOCKER_COMPOSE) run client1
docker-compose -f $(DOCKER_COMPOSE) down
# Like clustertests, but rebuilds all images.
clustertests-build: vendor
docker-compose -f $(DOCKER_COMPOSE) down -v
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
# Install Pilosa
install:

27
api.go
View file

@ -23,6 +23,7 @@ import (
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v2/rbf"
//"github.com/molecula/featurebase/v2/pg"
"github.com/molecula/featurebase/v2/pql"
@ -130,9 +131,16 @@ func (api *API) SetAPIOptions(opts ...apiOption) error {
var validAPIMethods = map[disco.ClusterState]map[apiMethod]struct{}{
disco.ClusterStateStarting: methodsCommon,
disco.ClusterStateNormal: appendMap(methodsCommon, methodsNormal),
disco.ClusterStateDegraded: appendMap(methodsCommon, methodsDegraded),
// Ideally, this would be just `appendMap(methodsCommon, methodsDegraded)`,
// but in an attempt to reduce the influence that state (determined by etcd)
// has on a node under load, this is set to effectively allow all requests
// in a DEGRADED state.
disco.ClusterStateDegraded: appendMap(methodsCommon, methodsNormal),
disco.ClusterStateResizing: appendMap(methodsCommon, methodsResizing),
disco.ClusterStateDown: methodsCommon,
// Ideally, this would be just `methodsCommon`, but in an attempt to reduce
// the influence that state (determined by etcd) has on a node under load,
// this is set to effectively allow all requests in a DOWN state.
disco.ClusterStateDown: appendMap(methodsCommon, methodsNormal),
}
func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} {
@ -3156,6 +3164,21 @@ func (api *API) Plan(ctx context.Context, q string) (*Stmt, error) {
return api.server.PlanSQL(ctx, q)
}
func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo {
infos := make(map[string]*rbf.DebugInfo)
for key, dbShard := range api.holder.Txf().dbPerShard.Flatmap {
wrapper, ok := dbShard.W.(*RbfDBWrapper)
if !ok {
continue
}
skey := fmt.Sprintf("%s/%d", key.index, key.shard)
infos[skey] = wrapper.db.DebugInfo()
}
return infos
}
type serverInfo struct {
ShardWidth uint64 `json:"shardWidth"`
ReplicaN int `json:"replicaN"`

22
api_test.go
View file

@ -1415,3 +1415,25 @@ func TestVariousApiTranslateCalls(t *testing.T) {
*/
}
}
func TestAPI_RBFDebugInfo(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
coord := c.GetPrimary()
if _, err := coord.API.CreateIndex(ctx, "i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if infos := coord.API.RBFDebugInfo(); infos == nil {
t.Fatal("expected info")
}
}

41
authn/authenticate.go
View file

@ -1,4 +1,6 @@
// Copyright 2021 Molecula Corp. All rights reserved.
// Package authn handles authentication
package authn
import (
@ -31,8 +33,8 @@ type Auth struct {
oAuthConfig *oauth2.Config
}
// NewAuth is a constructor that returns a new auth object
func NewAuth(logger logger.Logger, url string, scopes []string, authUrl, tokenUrl, groupEndpoint, logout, clientID, clientSecret, hashKey, blockKey string) (*Auth, error) {
// NewAuth instantiates and returns a new Auth struct
func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, hashKey, blockKey string) (*Auth, error) {
auth := &Auth{
logger: logger,
cookieName: "molecula-chip",
@ -46,8 +48,8 @@ func NewAuth(logger logger.Logger, url string, scopes []string, authUrl, tokenUr
ClientSecret: clientSecret,
Scopes: scopes,
Endpoint: oauth2.Endpoint{
AuthURL: authUrl,
TokenURL: tokenUrl,
AuthURL: authURL,
TokenURL: tokenURL,
},
},
}
@ -65,6 +67,7 @@ func NewAuth(logger logger.Logger, url string, scopes []string, authUrl, tokenUr
return auth, nil
}
// CookieValue holds the value of an authenticated user's cookie
type CookieValue struct {
UserID string
UserName string
@ -72,23 +75,24 @@ type CookieValue struct {
Token *oauth2.Token
}
type Groups struct {
Groups []Group `json:"value"`
}
// Group holds group information for an authenticated user
type Group struct {
UserID string
GroupID string `json:"id"`
GroupName string `json:"displayName"`
}
// UserInfo holds user information for an authenticated user
type UserInfo struct {
UserID string `json:"userid"`
UserName string `json:"username"`
}
// Authenticate reads and validates a cookie, redirects if invalid or missing, otherwise returns
// the group membership information stored in the cookie.
// Authenticate reads the authentication cookie from a request, returning the
// user's group memberships on success. If the cookie is not present or has expired,
// Authenticate redirects the user to sign in. If the cookie is within the
// refresh window of expiring, the cookie is refreshed, and the updated group
// membership is returned.
func (a *Auth) Authenticate(w http.ResponseWriter, r *http.Request) ([]Group, error) {
cookie, err := a.readCookie(w, r)
if err != nil {
@ -112,10 +116,10 @@ func (a *Auth) Authenticate(w http.ResponseWriter, r *http.Request) ([]Group, er
}
// Login redirects user to the IdP authorize endpoint for auth code
// Login redirects a user to login to their configured oAuth authorize endpoint
func (a *Auth) Login(w http.ResponseWriter, r *http.Request) {
authUrl := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
http.Redirect(w, r, authUrl, http.StatusTemporaryRedirect)
authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
}
// Logout clears out user cookie and redirects user to IdP's logout endpoint
@ -125,7 +129,8 @@ func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, redirect, http.StatusTemporaryRedirect)
}
// Redirect gets user information from IdP and sets a secure cookie
// Redirect handles the oAuth /redirect endpoint. It gets user information from
// the identity provider and sets a secure cookie holding the user information.
func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
code := r.FormValue("code")
token, err := a.getToken(r, code)
@ -154,11 +159,11 @@ func (a *Auth) GetUserInfo(w http.ResponseWriter, r *http.Request) *UserInfo {
a.logger.Warnf("was not able to read cookie for req: %+v", r)
return &resp
}
return &UserInfo{
UserID: cookie.UserID,
UserName: cookie.UserName,
}
}
// getToken exhanges authorization code for an oAuth2 token
@ -198,7 +203,7 @@ func (a *Auth) newCookieValue(token *oauth2.Token) (*CookieValue, error) {
return &CookieValue{
UserID: claims["oid"].(string),
UserName: claims["name"].(string),
GroupMembership: groups.Groups,
GroupMembership: groups,
Token: token,
}, nil
}
@ -206,6 +211,7 @@ func (a *Auth) newCookieValue(token *oauth2.Token) (*CookieValue, error) {
// getGroupMembership uses a oauth2 token to retrieve group membership information from IdP
func (a *Auth) getGroupMembership(token *oauth2.Token) (Groups, error) {
var groups Groups
req, err := http.NewRequest("GET", a.groupEndpoint, nil)
if err != nil {
return groups, errors.Wrap(err, "creating new request to group endpoint")
@ -267,7 +273,7 @@ func (a *Auth) setCookie(w http.ResponseWriter, cookie *CookieValue) error {
func (a *Auth) refreshToken(w http.ResponseWriter, cookie *CookieValue) error {
if cookie.Token.RefreshToken == "" {
return errors.New("no refresh token found, check auth scopes to see if refresh tokens are being provided by your IdP.")
return errors.New("no refresh token found, check auth scopes to see if refresh tokens are being provided by your IdP")
}
tokenSource := a.oAuthConfig.TokenSource(context.Background(), cookie.Token)
newToken, err := tokenSource.Token()
@ -307,4 +313,5 @@ func (a *Auth) getEmptyCookie() *http.Cookie {
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
}
}

6
authn/authenticate_internal_test.go
View file

@ -13,7 +13,7 @@ import (
func TestAuth(t *testing.T) {
var (
ClientId = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
ClientID = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
AuthorizeURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize"
TokenURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token"
@ -32,7 +32,7 @@ func TestAuth(t *testing.T) {
TokenURL,
GroupEndpointURL,
LogoutURL,
ClientId,
ClientID,
ClientSecret,
Key,
Key,
@ -94,7 +94,7 @@ func TestAuth(t *testing.T) {
TokenURL,
GroupEndpointURL,
LogoutURL,
ClientId,
ClientID,
ClientSecret,
Key,
ShortKey,

14
client.go
View file

@ -80,8 +80,14 @@ type InternalClient interface {
GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error)
GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error)
ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error
ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error
// ImportFieldKeys and ImportIndexKeys are mainly used when
// restoring a backup. They take a readerFunc which returns a
// reader rather than taking an io.Reader directly to allow for
// efficient retries (rather than reading the entire request body
// into a buffer and reusing it). Reader returned from the func
// must be properly closed by the implementation.
ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error
ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error
// SetInternalAPI tells the client the API it should use for internal/loopback ops
// where applicable.
@ -277,11 +283,11 @@ func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map
func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) {
return nil, nil
}
func (c nopInternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error {
func (c nopInternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error {
return nil
}
func (c nopInternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error {
func (c nopInternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error {
return nil
}

12
cmd/backup.go
View file

@ -23,11 +23,13 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
}
flags := ccmd.Flags()
flags.StringVarP(&cmd.OutputDir, "output", "o", "", "output dir to write to")
flags.BoolVar(&cmd.NoSync, "no-sync", false, "disable file sync")
flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "number of concurrent backup goroutines")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
flags.StringVar(&cmd.Index, "index", "", "index to backup, default backs up all indexes. ")
flags.StringVarP(&cmd.OutputDir, "output", "o", "", "Output directory to write to.")
flags.BoolVar(&cmd.NoSync, "no-sync", false, "Disable file sync")
flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "Number of concurrent backup goroutines.")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "The address (host:port) of FeatureBase (HTTP).")
flags.StringVar(&cmd.Index, "index", "", "Index to backup, default backs up all indexes. ")
flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
return ccmd
}

2
cmd/restore.go
View file

@ -25,6 +25,8 @@ The Restore command will take a backup archive and restore it to a new, clean cl
flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
flags.IntVar(&cmd.Concurrency, "concurrency", 1, "number of concurrent uploads")
flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
ctl.SetTLSConfig(
flags, "",
&cmd.TLS.CertificatePath,

12
cmd/slurp/slurp.go
View file

@ -92,8 +92,10 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
byteData, err := ioutil.ReadAll(tr)
vprint.PanicOn(err)
br := bytes.NewReader(byteData)
err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, br)
readerFunc := func() (io.Reader, error) {
return bytes.NewReader(byteData), nil
}
err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, readerFunc)
if err != nil {
return err
}
@ -106,9 +108,11 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
}
byteData, err := ioutil.ReadAll(tr)
vprint.PanicOn(err)
readerFunc := func() (io.Reader, error) {
return bytes.NewReader(byteData), nil
}
br := bytes.NewReader(byteData)
err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, br)
err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, readerFunc)
if err != nil {
return err
}

23
ctl/backup.go
View file

@ -10,11 +10,13 @@ import (
"io/ioutil"
"os"
"path/filepath"
"time"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/http"
fb_http "github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/topology"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -37,6 +39,12 @@ type BackupCommand struct { // nolint: maligned
// Number of concurrent backup goroutines running at a time.
Concurrency int
// Amount of time after first failed request to continue retrying.
RetryPeriod time.Duration `json:"retry-period"`
// Host:port on which to listen for pprof.
Pprof string `json:"pprof"`
// Reusable client.
client pilosa.InternalClient
@ -51,11 +59,20 @@ func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand
return &BackupCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
Concurrency: 1,
RetryPeriod: time.Minute,
Pprof: "localhost:43809",
}
}
// Run executes the main program execution.
func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
logger := cmd.Logger()
close, err := startProfilingServer(cmd.Pprof, logger)
if err != nil {
return errors.Wrap(err, "starting profiling server")
}
defer close()
// Validate arguments.
if cmd.OutputDir == "" {
return fmt.Errorf("-o flag required")
@ -70,7 +87,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
}
// Create a client to the server.
client, err := commandClient(cmd)
client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod))
if err != nil {
return fmt.Errorf("creating client: %w", err)
}
@ -262,7 +279,7 @@ func (cmd *BackupCommand) backupShardNode(ctx context.Context, indexName string,
logger := cmd.Logger()
logger.Printf("backing up shard: index=%q id=%d", indexName, shard)
client := http.NewInternalClientFromURI(&node.URI, http.GetHTTPClient(cmd.tlsConfig))
client := fb_http.NewInternalClientFromURI(&node.URI, fb_http.GetHTTPClient(cmd.tlsConfig), fb_http.WithClientRetryPeriod(cmd.RetryPeriod))
rc, err := client.ShardReader(ctx, indexName, shard)
if err != nil {
return fmt.Errorf("fetching shard reader: %w", err)

17
ctl/common.go
View file

@ -2,6 +2,11 @@
package ctl
import (
"net"
"time"
gohttp "net/http"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/server"
@ -25,14 +30,22 @@ func SetTLSConfig(flags *pflag.FlagSet, prefix string, certificatePath *string,
flags.BoolVarP(enableClientVerification, prefix+"tls.enable-client-verification", "", false, "Enable TLS certificate client verification for incoming connections")
}
// default dial timeout is 30s for some reason which makes testing
// failures/retries really awkward. I don't think we need it that
// high, so I set it to 1s here... let's see what happens.
func clientOptions(client *gohttp.Client, dialer *net.Dialer) *gohttp.Client {
dialer.Timeout = time.Second * 1
return client
}
// commandClient returns a pilosa.InternalHTTPClient for the command
func commandClient(cmd CommandWithTLSSupport) (*http.InternalClient, error) {
func commandClient(cmd CommandWithTLSSupport, opts ...http.InternalClientOption) (*http.InternalClient, error) {
tls := cmd.TLSConfiguration()
tlsConfig, err := server.GetTLSConfig(&tls, cmd.Logger())
if err != nil {
return nil, errors.Wrap(err, "getting tls config")
}
client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(tlsConfig))
client, err := http.NewInternalClient(cmd.TLSHost(), http.GetHTTPClient(tlsConfig, clientOptions), opts...)
if err != nil {
return nil, errors.Wrap(err, "getting internal client")
}

12
ctl/rbf_check.go
View file

@ -26,7 +26,7 @@ func NewRBFCheckCommand(stdin io.Reader, stdout, stderr io.Writer) *RBFCheckComm
}
}
// Run executes the export.
// Run executes a consistency check of an RBF database.
func (cmd *RBFCheckCommand) Run(ctx context.Context) error {
// Open database.
db := rbf.NewDB(cmd.Path, nil)
@ -37,7 +37,15 @@ func (cmd *RBFCheckCommand) Run(ctx context.Context) error {
// Run check on the database.
if err := db.Check(); err != nil {
return err
switch err := err.(type) {
case rbf.ErrorList:
for i := range err {
fmt.Fprintln(cmd.Stdout, err[i])
}
default:
fmt.Fprintln(cmd.Stdout, err)
}
return fmt.Errorf("check failed")
}
// If successful, print a success message.

33
ctl/rbf_check_test.go Normal file
View file

@ -0,0 +1,33 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
"bytes"
"context"
"path/filepath"
"testing"
)
func TestRBFCheckCommand_Run(t *testing.T) {
t.Run("OK", func(t *testing.T) {
var stdout, stderr bytes.Buffer
cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr)
cmd.Path = filepath.Join("testdata", "rbf-check", "ok")
if err := cmd.Run(context.Background()); err != nil {
t.Fatal(err)
} else if got, want := stdout.String(), `ok`+"\n"; got != want {
t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
}
})
t.Run("ErrInvalidPageType", func(t *testing.T) {
var stdout, stderr bytes.Buffer
cmd := NewRBFCheckCommand(bytes.NewReader(nil), &stdout, &stderr)
cmd.Path = filepath.Join("testdata", "rbf-check", "err-invalid-page-type")
if err := cmd.Run(context.Background()); err == nil || err.Error() != `check failed` {
t.Fatal(err)
} else if got, want := stdout.String(), `page not in-use & not free: pgno=4`+"\n"; got != want {
t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
}
})
}

11
ctl/rbf_pages.go
View file

@ -49,7 +49,16 @@ func (cmd *RBFPagesCommand) Run(ctx context.Context) error {
// Iterate over each page and grab info.
infos, err := tx.PageInfos()
if err != nil {
return err
fmt.Fprintln(cmd.Stdout, "ERRORS:")
switch err := err.(type) {
case rbf.ErrorList:
for i := range err {
fmt.Fprintln(cmd.Stdout, err[i])
}
default:
fmt.Fprintln(cmd.Stdout, err)
}
fmt.Fprintln(cmd.Stdout, "")
}
// Write header.

52
ctl/rbf_pages_test.go Normal file
View file

@ -0,0 +1,52 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
"bytes"
"context"
"path/filepath"
"testing"
)
func TestRBFPagesCommand_Run(t *testing.T) {
t.Run("OK", func(t *testing.T) {
want := `
ID TYPE EXTRA
======== ========== ====================
0 meta pageN=4,walid=4,rootrec=1,freelist=2
1 rootrec next=0
2 leaf flags=x2,celln=0
3 leaf flags=x2,celln=1
`[1:]
var stdout, stderr bytes.Buffer
cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr)
cmd.Path = filepath.Join("testdata", "rbf-pages", "ok")
if err := cmd.Run(context.Background()); err != nil {
t.Fatal(err)
} else if got := stdout.String(); got != want {
t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
}
})
t.Run("ErrInvalidPageType", func(t *testing.T) {
want := `
ID TYPE EXTRA
======== ========== ====================
0 meta pageN=5,walid=4,rootrec=1,freelist=2
1 rootrec next=0
2 leaf flags=x2,celln=0
3 leaf flags=x2,celln=1
4 unknown [<nil>]
`[1:]
var stdout, stderr bytes.Buffer
cmd := NewRBFPagesCommand(bytes.NewReader(nil), &stdout, &stderr)
cmd.Path = filepath.Join("testdata", "rbf-pages", "err-invalid-page-type")
if err := cmd.Run(context.Background()); err != nil {
t.Fatal(err)
} else if got := stdout.String(); got != want {
t.Fatalf("got:\n%s\n\nwant:\n%s", got, want)
}
})
}

75
ctl/restore.go
View file

@ -5,18 +5,23 @@ import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/hashicorp/go-retryablehttp"
pilosa "github.com/molecula/featurebase/v2"
fb_http "github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/topology"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -29,6 +34,13 @@ type RestoreCommand struct {
// Filepath to the backup file.
Path string
// Amount of time after first failed request to continue retrying.
RetryPeriod time.Duration `json:"retry-period"`
// Host:port on which to listen for pprof.
Pprof string `json:"pprof"`
// Reusable client.
client pilosa.InternalClient
@ -41,13 +53,20 @@ type RestoreCommand struct {
func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
return &RestoreCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
RetryPeriod: time.Second * 30,
Concurrency: 1,
Pprof: "localhost:43809",
}
}
// Run executes the restore.
func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
logger := cmd.Logger()
close, err := startProfilingServer(cmd.Pprof, logger)
if err != nil {
return errors.Wrap(err, "starting profiling server")
}
defer close()
// Validate arguments.
if cmd.Path == "" {
@ -62,7 +81,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
return fmt.Errorf("parsing tls config: %w", err)
}
// Create a client to the server.
client, err := commandClient(cmd)
client, err := commandClient(cmd, fb_http.WithClientRetryPeriod(cmd.RetryPeriod))
if err != nil {
return fmt.Errorf("creating client: %w", err)
}
@ -119,7 +138,7 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology.
if len(existingSchema) == 0 {
cmd.Logger().Printf("Load Schema")
url := primary.URI.Path("/schema")
var client http.Client
client := cmd.newClient()
_, err = client.Post(url, "application/json", f)
} else {
schema := &pilosa.Schema{}
@ -159,6 +178,34 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology.
return err
}
func retryWith400(ctx context.Context, resp *http.Response, err error) (bool, error) {
if resp != nil && resp.StatusCode >= 400 { // we have some dumb status codes
return true, nil
}
return retryablehttp.DefaultRetryPolicy(ctx, resp, err)
}
// This logic is taken from featurebase/http/client.go If this logic
// is not the same as what's there, that could be a problem. Ideally
// all network calls from restore would go through the client and this
// would not longer be needed.
func (cmd *RestoreCommand) newClient() *retryablehttp.Client {
min := time.Millisecond * 100
// do some math to figure out how many attempts we need to get our
// total sleep time close to the period
attempts := math.Log2(float64(cmd.RetryPeriod)) - math.Log2(float64(min))
attempts += 0.3 // mmmm, fudge
if attempts < 1 {
attempts = 1
}
client := retryablehttp.NewClient()
client.RetryWaitMin = min
client.RetryMax = int(attempts)
client.CheckRetry = retryWith400
return client
}
func (cmd *RestoreCommand) restoreIDAlloc(ctx context.Context, primary *topology.Node) error {
logger := cmd.Logger()
@ -174,7 +221,7 @@ func (cmd *RestoreCommand) restoreIDAlloc(ctx context.Context, primary *topology
logger.Printf("Load idalloc")
url := primary.URI.Path("/internal/idalloc/restore")
var client http.Client
client := cmd.newClient()
_, err = client.Post(url, "application/octet-stream", f)
return err
}
@ -244,14 +291,14 @@ func (cmd *RestoreCommand) restoreShard(ctx context.Context, filename string) er
defer f.Close()
url := node.URI.Path(fmt.Sprintf("/internal/restore/%v/%v", indexName, shard))
req, err := http.NewRequest("POST", url, f)
req, err := retryablehttp.NewRequest("POST", url, f)
if err != nil {
return err
}
req = req.WithContext(ctx)
req.Header.Set("Content-Type", "application/octet-stream")
var client http.Client
client := cmd.newClient()
resp, err := client.Do(req)
if err != nil {
return err
@ -319,13 +366,11 @@ func (cmd *RestoreCommand) restoreIndexTranslationFile(ctx context.Context, file
for _, node := range nodes {
if err := func() error {
f, err := os.Open(filename)
if err != nil {
return err
readerFunc := func() (io.Reader, error) {
return os.Open(filename) // gets used as an HTTP request body and closed by http library
}
defer f.Close()
return cmd.client.ImportIndexKeys(ctx, &node.URI, indexName, partitionID, false, f)
return cmd.client.ImportIndexKeys(ctx, &node.URI, indexName, partitionID, false, readerFunc)
}(); err != nil {
return err
}
@ -380,13 +425,11 @@ func (cmd *RestoreCommand) restoreFieldTranslationFile(ctx context.Context, node
for _, node := range nodes {
if err := func() error {
f, err := os.Open(filename)
if err != nil {
return err
readerFunc := func() (io.Reader, error) {
return os.Open(filename)
}
defer f.Close()
return cmd.client.ImportFieldKeys(ctx, &node.URI, indexName, fieldName, false, f)
return cmd.client.ImportFieldKeys(ctx, &node.URI, indexName, fieldName, false, readerFunc)
}(); err != nil {
return err
}

2
ctl/server.go
View file

@ -45,7 +45,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
// Etcd
// Etcd.Name used Config.Name for its value.
// Etcd.Dir defaults to a directory under the pilosa data directory.
flags.StringVar(&srv.Config.Etcd.Dir, "etcd.dir", srv.Config.Etcd.Dir, "Directory to store etcd data files. If not provided, a directory will be created under the main data-dir directory.")
// Etcd.ClusterName uses Cluster.Name for its value
flags.StringVar(&srv.Config.Etcd.LClientURL, "etcd.listen-client-address", srv.Config.Etcd.LClientURL, "Listen client address.")
flags.StringVar(&srv.Config.Etcd.AClientURL, "etcd.advertise-client-address", srv.Config.Etcd.AClientURL, "Advertise client address. If not provided, uses the listen client address.")

BIN
ctl/testdata/rbf-check/err-invalid-page-type/data vendored Normal file
View file

Binary file not shown.

0
ctl/testdata/rbf-check/err-invalid-page-type/wal vendored Normal file
View file

BIN
ctl/testdata/rbf-check/ok/data vendored Normal file
View file

Binary file not shown.

0
ctl/testdata/rbf-check/ok/wal vendored Normal file
View file

BIN
ctl/testdata/rbf-pages/err-invalid-page-type/data vendored Normal file
View file

Binary file not shown.

0
ctl/testdata/rbf-pages/err-invalid-page-type/wal vendored Normal file
View file

BIN
ctl/testdata/rbf-pages/ok/data vendored Normal file
View file

Binary file not shown.

0
ctl/testdata/rbf-pages/ok/wal vendored Normal file
View file

56
ctl/util.go Normal file
View file

@ -0,0 +1,56 @@
package ctl
import (
"context"
"net"
"net/http"
"net/http/pprof"
"runtime"
"time"
"github.com/felixge/fgprof"
"github.com/molecula/featurebase/v2/logger"
"github.com/pkg/errors"
)
// startProfilingServer starts a server which handles /debug/pprof and
// /debug/fgprof for use in utilities we might want to profile but
// wouldn't otherwise be running an http server. Caller should call
// the returned close function before exiting to release resources.
func startProfilingServer(addr string, logger logger.Logger) (close func() error, err error) {
if addr == "" {
return func() error { return nil }, nil
}
sm := http.NewServeMux()
sm.Handle("/debug/fgprof", fgprof.Handler())
sm.HandleFunc("/debug/pprof/", pprof.Index)
sm.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
sm.HandleFunc("/debug/pprof/profile", pprof.Profile)
sm.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
sm.HandleFunc("/debug/pprof/trace", pprof.Trace)
s := &http.Server{
Addr: addr,
Handler: sm,
}
runtime.SetBlockProfileRate(10000000) // 1 sample per 10 ms
runtime.SetMutexProfileFraction(100) // 1% sampling
ln, err := net.Listen("tcp", addr)
if err != nil {
return nil, err
}
go func() {
logger.Printf("Listening for /debug/pprof/ and /debug/fgprof on '%s'", addr)
logger.Printf("%v", s.Serve(ln))
}()
return func() error {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
err := s.Shutdown(ctx)
if err != nil {
return errors.Wrap(err, "shutting down profiling server")
}
return s.Close()
}, nil
}

53
executor.go
View file

@ -179,11 +179,18 @@ func newExecutor(opts ...executorOption) *executor {
func (e *executor) addWorker() {
e.workersWG.Add(1)
atomic.AddInt64(&e.currentWorkers, 1)
n := atomic.AddInt64(&e.currentWorkers, 1)
if e.Holder != nil {
e.Holder.Stats.Gauge("worker_total", float64(n), 0)
}
go func() {
defer e.workersWG.Done()
e.worker(e.work)
atomic.AddInt64(&e.currentWorkers, -1)
n := atomic.AddInt64(&e.currentWorkers, -1)
if e.Holder != nil {
e.Holder.Stats.Gauge("worker_total", float64(n), 0)
}
}()
}
@ -204,6 +211,14 @@ func (e *executor) Close() error {
return nil
}
// InitStats initializes stats counters. Must be called after Holder set.
func (e *executor) InitStats() {
if e.Holder != nil {
e.Holder.Stats.Count("job_total", 0, 0)
e.Holder.Stats.Gauge("worker_total", float64(atomic.LoadInt64(&e.currentWorkers)), 0)
}
}
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute")
@ -587,6 +602,11 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q
return nil, err
}
if vc, ok := v.(ValCount); ok {
vc.cleanup()
v = vc
}
results = append(results, v)
// Some Calls can have significant data associated with them
// that gets generated during processing, such as Precomputed
@ -597,6 +617,22 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q
return results, nil
}
// cleanup removes the integer value (Val) from the ValCount if one of
// the other fields is in use.
//
// ValCounts are normally holding data which is stored as a BSI
// (integer) under the hood. Sometimes it's convenient to be able to
// compare the underlying integer values rather than their
// interpretation as decimal, timestamp, etc, so the lower level
// functions may return both integer and the interpreted value, but we
// don't want to pass that all the way back to the client, so we
// remove it here.
func (vc *ValCount) cleanup() {
if vc.Val != 0 && (vc.FloatVal != 0 || !vc.TimestampVal.IsZero() || vc.DecimalVal != nil) {
vc.Val = 0
}
}
// preprocessQuery expands any calls that need preprocessing.
func (e *executor) preprocessQuery(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*pql.Call, error) {
switch c.Name {
@ -1261,6 +1297,10 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string
if err != nil {
return ValCount{}, errors.New("Percentile(): field required")
}
field := e.Holder.Field(index, fieldName)
if field == nil {
return ValCount{}, ErrFieldNotFound
}
// filter call for min & max
var filterCall *pql.Call
@ -1281,7 +1321,7 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string
return ValCount{}, errors.Wrap(err, "executing Min call for Percentile")
}
if nthFloat == 0.0 {
return ValCount{Val: minVal.Val, Count: minVal.Count}, nil
return minVal, nil
}
// get max
@ -1348,11 +1388,11 @@ func (e *executor) executePercentile(ctx context.Context, qcx *Qcx, index string
} else if leftCountWeighted < rightCount {
min = possibleNthVal + 1
} else {
return ValCount{Val: possibleNthVal, Count: 1}, nil
return field.valCountize(possibleNthVal, 1, nil)
}
}
return ValCount{Val: min, Count: 1}, nil
return field.valCountize(min, 1, nil)
}
@ -5989,6 +6029,7 @@ type job struct {
func (e *executor) worker(work chan job) {
for j := range work {
atomic.AddUint64(&e.workCounter, 1)
e.Holder.Stats.Count("job_total", 1, 0)
if j.idleHands {
return
}
@ -8130,6 +8171,8 @@ func getScaledInt(f *Field, v interface{}) (int64, error) {
switch tv := v.(type) {
case time.Time:
value = tv.UnixNano() / TimeUnitNanos(f.options.TimeUnit)
case int64:
value = tv
default:
return 0, errors.Errorf("unexpected timestamp value type %T, val %v", tv, tv)
}

15
executor_internal_test.go
View file

@ -489,3 +489,18 @@ func TestExecutorSafeCopyDistinctTimestamp(t *testing.T) {
t.Fatalf("Did not copy results. got %+v, want %+v", copied.Results, response.Results)
}
}
func TestGetScaledInt(t *testing.T) {
f := OpenField(t, OptFieldTypeTimestamp(time.Now(), "ms"))
defer f.Close()
// check that fields with type timestamp return the int64 passed in to getScaledInt with nil err
v := time.Now().Unix()
res, err := getScaledInt(f.Field, v)
if err != nil {
t.Errorf("got error %v, expected nil", err)
}
if !reflect.DeepEqual(res, v) {
t.Errorf("expected %v, got %v", v, res)
}
}

38
field.go
View file

@ -1388,18 +1388,7 @@ func (f *Field) MaxForShard(tx Tx, shard uint64, filter *Row) (ValCount, error)
return ValCount{}, errors.Wrap(err, "calling fragment.max")
}
valCount := ValCount{Count: int64(cnt)}
if f.Options().Type == FieldTypeDecimal {
dec := pql.NewDecimal(max+bsig.Base, bsig.Scale)
valCount.DecimalVal = &dec
} else if f.Options().Type == FieldTypeTimestamp {
valCount.TimestampVal = time.Unix(0, (max+bsig.Base)*TimeUnitNanos(f.options.TimeUnit)).UTC()
} else {
valCount.Val = max + bsig.Base
}
return valCount, nil
return f.valCountize(max, cnt, bsig)
}
// MinForShard returns the minimum value which appears in this shard
@ -1434,17 +1423,32 @@ func (f *Field) MinForShard(tx Tx, shard uint64, filter *Row) (ValCount, error)
return ValCount{}, errors.Wrap(err, "calling fragment.min")
}
return f.valCountize(min, cnt, bsig)
}
// valCountize takes the "raw" value and count we get from the
// fragment and calculates the cooked values for this field
// (timestamping, decimaling, or just adding in the base). It always
// includes the int64 "Val\" value to make comparisons easier in the
// executor (at time of writing, Percentile takes advantage of this,
// but we might be able to simplify logic in other places as well).
func (f *Field) valCountize(val int64, cnt uint64, bsig *bsiGroup) (ValCount, error) {
if bsig == nil {
bsig = f.bsiGroup(f.name)
if bsig == nil {
return ValCount{}, ErrBSIGroupNotFound
}
}
valCount := ValCount{Count: int64(cnt)}
if f.Options().Type == FieldTypeDecimal {
dec := pql.NewDecimal(min+bsig.Base, bsig.Scale)
dec := pql.NewDecimal(val+bsig.Base, bsig.Scale)
valCount.DecimalVal = &dec
} else if f.Options().Type == FieldTypeTimestamp {
valCount.TimestampVal = time.Unix(0, (min+bsig.Base)*TimeUnitNanos(f.options.TimeUnit)).UTC()
} else {
valCount.Val = min + bsig.Base
valCount.TimestampVal = time.Unix(0, (val+bsig.Base)*TimeUnitNanos(f.options.TimeUnit)).UTC()
}
valCount.Val = val + bsig.Base
return valCount, nil
}

33
field_internal_test.go
View file

@ -182,6 +182,23 @@ func TestBSIGroup_BaseValue(t *testing.T) {
})
}
func TestField_ValCountize(t *testing.T) {
f := OpenField(t, OptFieldTypeDefault())
defer f.Close()
// check that you get an empty val count and err
// BSIGroupNotFound on nil bsig from
// f.bsiGroup(f.name)
f.bsiGroups = []*bsiGroup{}
v, err := f.valCountize(42, 42, nil)
if !reflect.DeepEqual(v, ValCount{}) {
t.Errorf("expected %v, got %v", ValCount{}, v)
}
if err != ErrBSIGroupNotFound {
t.Errorf("expected %v, got %v", ErrBSIGroupNotFound, err)
}
}
// Ensure field can open and retrieve a view.
func TestField_DeleteView(t *testing.T) {
f := OpenField(t, OptFieldTypeDefault())
@ -748,29 +765,29 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
name: "single",
columnIDs: []uint64{1},
values: []float64{10.1},
expMax: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
expMin: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
expMax: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
expMin: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
},
{
name: "twovals",
columnIDs: []uint64{1, 2},
values: []float64{10.1, 20.2},
expMax: ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1},
expMin: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
expMax: ValCount{Val: 20200, DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 1},
expMin: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 1},
},
{
name: "multiplecounts",
columnIDs: []uint64{1, 2, 3, 4, 5},
values: []float64{10.1, 20.2, 10.1, 10.1, 20.2},
expMax: ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
expMin: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
expMax: ValCount{Val: 20200, DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
expMin: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
},
{
name: "middlevals",
columnIDs: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
values: []float64{10.1, 20.2, 10.1, 10.1, 20.2, 11, 12, 11, 13, 11},
expMax: ValCount{DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
expMin: ValCount{DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
expMax: ValCount{Val: 20200, DecimalVal: &pql.Decimal{Value: 20200, Scale: 3}, Count: 2},
expMin: ValCount{Val: 10100, DecimalVal: &pql.Decimal{Value: 10100, Scale: 3}, Count: 3},
},
} {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {

1
go.mod
View file

@ -27,6 +27,7 @@ require (
github.com/gorilla/handlers v1.3.0
github.com/gorilla/mux v1.7.0
github.com/gorilla/securecookie v1.1.1
github.com/hashicorp/go-retryablehttp v0.7.0
github.com/improbable-eng/grpc-web v0.13.0
github.com/lib/pq v1.8.0
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b

5
go.sum
View file

@ -190,10 +190,15 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=

111
http/client.go
View file

@ -8,18 +8,22 @@ import (
"fmt"
"io"
"io/ioutil"
"math"
"math/rand"
"net/http"
"net/url"
"os"
"path"
"sort"
"strconv"
"strings"
"time"
"github.com/hashicorp/go-retryablehttp"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/encoding/proto"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v2/logger"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/topology"
"github.com/molecula/featurebase/v2/tracing"
@ -31,8 +35,11 @@ type InternalClient struct {
defaultURI *pnet.URI
serializer pilosa.Serializer
log logger.Logger
// The client to use for HTTP communication.
httpClient *http.Client
httpClient *http.Client
retryableClient *retryablehttp.Client
// the local node's API, used for operations that we can short-circuit that way
api *pilosa.API
}
@ -40,7 +47,7 @@ type InternalClient struct {
// NewInternalClient returns a new instance of InternalClient to connect to host.
// If api is non-nil, the client uses it for some same-host operations instead
// of going through http.
func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient, error) {
func NewInternalClient(host string, remoteClient *http.Client, opts ...InternalClientOption) (*InternalClient, error) {
if host == "" {
return nil, pilosa.ErrHostRequired
}
@ -50,16 +57,75 @@ func NewInternalClient(host string, remoteClient *http.Client) (*InternalClient,
return nil, errors.Wrap(err, "getting URI")
}
client := NewInternalClientFromURI(uri, remoteClient)
client := NewInternalClientFromURI(uri, remoteClient, opts...)
return client, nil
}
func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client) *InternalClient {
return &InternalClient{
type InternalClientOption func(c *InternalClient)
// WithClientRetryPeriod is the max amount of total time the client will
// retry failed requests using exponential backoff.
func WithClientRetryPeriod(period time.Duration) InternalClientOption {
min := time.Millisecond * 100
// do some math to figure out how many attempts we need to get our
// total sleep time close to the period
attempts := math.Log2(float64(period)) - math.Log2(float64(min))
attempts += 0.3 // mmmm, fudge
if attempts < 1 {
attempts = 1
}
fmt.Println("attempts: ", int(attempts))
return func(c *InternalClient) {
rc := retryablehttp.NewClient()
rc.HTTPClient = c.httpClient
rc.RetryWaitMin = min
rc.RetryMax = int(attempts)
rc.CheckRetry = retryWith400Policy
c.retryableClient = rc
}
}
func WithClientLogger(log logger.Logger) InternalClientOption {
return func(c *InternalClient) {
c.log = log
}
}
func noRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) {
return false, nil
}
// retryWith400Policy wraps retryablehttp's default retry policy to
// also retry on 4XX errors which *should* be client errors and
// therefore useless to retry, but we have some incorrect status codes.
// TODO: fix the incorrect status codes so we can get rid of this.
func retryWith400Policy(ctx context.Context, resp *http.Response, err error) (bool, error) {
if resp != nil && resp.StatusCode >= 400 {
return true, nil
}
return retryablehttp.DefaultRetryPolicy(ctx, resp, err)
}
func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client, opts ...InternalClientOption) *InternalClient {
ic := &InternalClient{
defaultURI: defaultURI,
serializer: proto.Serializer{},
httpClient: remoteClient,
log: logger.NewStandardLogger(os.Stderr),
}
for _, opt := range opts {
opt(ic)
}
if ic.retryableClient == nil {
rc := retryablehttp.NewClient()
rc.HTTPClient = ic.httpClient
rc.CheckRetry = noRetryPolicy
ic.retryableClient = rc
}
return ic
}
// MaxShardByIndex returns the number of shards on a server by index.
@ -1717,19 +1783,36 @@ func giveRawResponse(b bool) executeRequestOption {
}
}
type nopCloser struct {
*bytes.Reader
}
func (n nopCloser) Close() error {
return nil
}
// executeRequest executes the given request and checks the Response. For
// responses with non-2XX status, the body is read and closed, and an error is
// returned. If the error is nil, the caller must ensure that the response body
// is closed.
func (c *InternalClient) executeRequest(req *http.Request, opts ...executeRequestOption) (*http.Response, error) {
return c.executeRetryableRequest(&retryablehttp.Request{Request: req}, opts...)
}
func (c *InternalClient) executeRetryableRequest(req *retryablehttp.Request, opts ...executeRequestOption) (*http.Response, error) {
tracing.GlobalTracer.InjectHTTPHeaders(req.Request)
req.Close = false
eo := &executeOpts{}
for _, opt := range opts {
opt(eo)
}
tracing.GlobalTracer.InjectHTTPHeaders(req)
req.Close = false
resp, err := c.httpClient.Do(req)
resp, err := c.retryableClient.Do(req)
return c.handleResponse(req.Request, eo, resp, err)
}
func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp *http.Response, err error) (*http.Response, error) {
if err != nil {
if resp != nil {
resp.Body.Close()
@ -2009,7 +2092,7 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context,
return resp.Body, nil
}
func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error {
func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, readerFunc func() (io.Reader, error)) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportIndexKeys")
defer span.Finish()
@ -2026,14 +2109,14 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, ind
url := fmt.Sprintf("%s/internal/translate/index/%s/%d", uri, index, partitionID)
// Generate HTTP request.
httpReq, err := http.NewRequest("POST", url, rddbdata)
httpReq, err := retryablehttp.NewRequest("POST", url, readerFunc)
if err != nil {
return errors.Wrap(err, "creating request")
}
httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.executeRequest(httpReq.WithContext(ctx))
resp, err := c.executeRetryableRequest(httpReq.WithContext(ctx))
if err != nil {
return err
}
@ -2041,7 +2124,7 @@ func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, ind
return nil
}
func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error {
func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, readerFunc func() (io.Reader, error)) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportFieldKeys")
defer span.Finish()
@ -2058,14 +2141,14 @@ func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, ind
url := fmt.Sprintf("%s/internal/translate/field/%s/%s", uri, index, field)
// Generate HTTP request.
httpReq, err := http.NewRequest("POST", url, rddbdata)
httpReq, err := retryablehttp.NewRequest("POST", url, readerFunc)
if err != nil {
return errors.Wrap(err, "creating request")
}
httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.executeRequest(httpReq.WithContext(ctx))
resp, err := c.executeRetryableRequest(httpReq.WithContext(ctx))
if err != nil {
return err
}

41
http/handler.go
View file

@ -479,6 +479,9 @@ func newRouter(handler *Handler) http.Handler {
router.HandleFunc("/internal/idalloc/data", handler.chkAuthN(handler.handleIDAllocData)).Methods("GET").Name("IDAllocData")
router.HandleFunc("/internal/restore/{index}/{shardID}", handler.chkAuthN(handler.handlePostRestore)).Methods("POST").Name("Restore")
router.HandleFunc("/internal/debug/rbf", handler.chkAuthN(handler.handleGetInternalDebugRBFJSON)).Methods("GET").Name("GetInternalDebugRBFJSON")
// endpoints for collecting cpu profiles from a chosen begin point to
// when the client wants to stop. Used for profiling imports that
// could be long or short.
@ -2206,6 +2209,18 @@ func validateProtobufHeader(r *http.Request) (error string, code int) {
return
}
// handleGetInternalDebugRBFJSON handles /internal/debug/rbf requests.
func (h *Handler) handleGetInternalDebugRBFJSON(w http.ResponseWriter, r *http.Request) {
buf, err := json.MarshalIndent(h.api.RBFDebugInfo(), "", " ")
if err != nil {
http.Error(w, "marshal json: "+err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(buf)
}
// handleGetMetricsJSON handles /metrics.json requests, translating text metrics results to more consumable JSON.
func (h *Handler) handleGetMetricsJSON(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
@ -2722,14 +2737,17 @@ func (s queryValidationSpec) validate(query url.Values) error {
return nil
}
func GetHTTPClient(t *tls.Config) *http.Client {
type ClientOption func(client *http.Client, dialer *net.Dialer) *http.Client
func GetHTTPClient(t *tls.Config, opts ...ClientOption) *http.Client {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
Proxy: http.ProxyFromEnvironment,
DialContext: dialer.DialContext,
MaxIdleConns: 1000,
MaxIdleConnsPerHost: 200,
IdleConnTimeout: 90 * time.Second,
@ -2739,7 +2757,12 @@ func GetHTTPClient(t *tls.Config) *http.Client {
if t != nil {
transport.TLSClientConfig = t
}
return &http.Client{Transport: transport}
client := &http.Client{Transport: transport}
for _, opt := range opts {
client = opt(client, dialer)
}
return client
}
// handlePostImportAtomicRecord handles /import-atomic-record requests
@ -3492,7 +3515,7 @@ func (h *Handler) handlePostRestore(w http.ResponseWriter, r *http.Request) {
//validate shard for this node
err = h.api.RestoreShard(ctx, indexName, shard, r.Body)
if err != nil {
http.Error(w, fmt.Sprintf("failed to restore shared %v %v err:%v", indexName, shard, err), http.StatusBadRequest)
http.Error(w, fmt.Sprintf("failed to restore shard %v %v err:%v", indexName, shard, err), http.StatusBadRequest)
return
}

158
internal/clustertests/cluster_test.go
View file

@ -3,6 +3,8 @@ package clustertest
import (
"context"
"fmt"
"net/http"
"os"
"os/exec"
"testing"
@ -30,56 +32,53 @@ func TestClusterStuff(t *testing.T) {
t.Fatalf("getting client: %v", err)
}
t.Run("long pause", func(t *testing.T) {
err := cli1.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
err = cli1.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100})
if err != nil {
t.Fatalf("creating field: %v", err)
}
if err := cli1.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{}); err != nil {
t.Fatalf("creating index: %v", err)
}
if err := cli1.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100}); err != nil {
t.Fatalf("creating field: %v", err)
}
req := &pilosa.ImportRequest{
Index: "testidx",
Field: "testf",
}
req.ColumnIDs = make([]uint64, 10)
req.RowIDs = make([]uint64, 10)
req := &pilosa.ImportRequest{
Index: "testidx",
Field: "testf",
}
req.ColumnIDs = make([]uint64, 10)
req.RowIDs = make([]uint64, 10)
for i := 0; i < 1000; i++ {
req.RowIDs[i%10] = 0
req.ColumnIDs[i%10] = uint64((i/10)*pilosa.ShardWidth + i%10)
req.Shard = uint64(i / 10)
if i%10 == 9 {
err = cli1.Import(context.Background(), nil, req, &pilosa.ImportOptions{})
if err != nil {
t.Fatalf("importing: %v", err)
}
}
}
// Check query results from each node.
for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} {
r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
for i := 0; i < 1000; i++ {
req.RowIDs[i%10] = 0
req.ColumnIDs[i%10] = uint64((i/10)*pilosa.ShardWidth + i%10)
req.Shard = uint64(i / 10)
if i%10 == 9 {
err = cli1.Import(context.Background(), nil, req, &pilosa.ImportOptions{})
if err != nil {
t.Fatalf("count querying pilosa%d: %v", i, err)
}
if r.Results[0].(uint64) != 1000 {
t.Fatalf("count on pilosa%d after import is %d", i, r.Results[0].(uint64))
t.Fatalf("importing: %v", err)
}
}
}
// Check query results from each node.
for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} {
r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
if err != nil {
t.Fatalf("count querying pilosa%d: %v", i, err)
}
if r.Results[0].(uint64) != 1000 {
t.Fatalf("count on pilosa%d after import is %d", i, r.Results[0].(uint64))
}
}
t.Run("long pause", func(t *testing.T) {
pcmd := exec.Command("/pumba", "pause", "clustertests_pilosa3_1", "--duration", "10s")
pcmd.Stdout = os.Stdout
pcmd.Stderr = os.Stderr
t.Log("pausing pilosa3 for 10s")
err = pcmd.Start()
if err != nil {
if err := pcmd.Start(); err != nil {
t.Fatalf("starting pumba command: %v", err)
}
err = pcmd.Wait()
if err != nil {
if err := pcmd.Wait(); err != nil {
t.Fatalf("waiting on pumba pause cmd: %v", err)
}
@ -98,6 +97,89 @@ func TestClusterStuff(t *testing.T) {
}
}
})
t.Run("backup", func(t *testing.T) {
// do backup with node 1 down, but restart it after a few seconds
if err := sendCmd("docker", "stop", "clustertests_pilosa1_1"); err != nil {
t.Fatalf("sending stop command: %v", err)
}
var backupCmd *exec.Cmd
tmpdir := t.TempDir()
if backupCmd, err = startCmd(
"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest")); err != nil {
t.Fatalf("sending backup command: %v", err)
}
time.Sleep(time.Second * 5)
if err = sendCmd("docker", "start", "clustertests_pilosa1_1"); err != nil {
t.Fatalf("sending start command: %v", err)
}
if err = backupCmd.Wait(); err != nil {
t.Fatalf("waiting on backup to finish: %v", err)
}
fmt.Println("STARTING RESTORE")
client := http.Client{}
if req, err := http.NewRequest(http.MethodDelete, "http://pilosa1:10101/index/testidx", nil); err != nil {
t.Fatalf("getting req: %v", err)
} else if resp, err := client.Do(req); err != nil {
t.Fatalf("doing request: %v", err)
} else if resp.StatusCode >= 400 {
t.Fatalf("bad response: %v", resp)
}
var restoreCmd *exec.Cmd
if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101"); err != nil {
t.Fatalf("starting restore: %v", err)
}
time.Sleep(time.Millisecond * 50)
if err = sendCmd("docker", "stop", "clustertests_pilosa2_1"); err != nil {
t.Fatalf("sending stop command: %v", err)
}
time.Sleep(time.Second * 10)
if err = sendCmd("docker", "start", "clustertests_pilosa2_1"); err != nil {
t.Fatalf("sending stop command: %v", err)
}
if err := restoreCmd.Wait(); err != nil {
t.Fatalf("restore failed: %v", err)
}
// now do backup with all nodes down and too short a timeout
// so it fails. Has be to be all 3 because the cluster has
// replicas=3 and the backup command will retry on replicas.
if backupCmd, err = startCmd(
"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms"); err != nil {
t.Fatalf("sending second backup command: %v", err)
}
time.Sleep(time.Millisecond * 10) // want the backup to get started, then fail
if err = sendCmd("docker", "stop", "clustertests_pilosa1_1"); err != nil {
t.Fatalf("sending stop command: %v", err)
}
if err = sendCmd("docker", "stop", "clustertests_pilosa2_1"); err != nil {
t.Fatalf("sending stop command: %v", err)
}
if err = sendCmd("docker", "stop", "clustertests_pilosa3_1"); err != nil {
t.Fatalf("sending stop command: %v", err)
}
time.Sleep(time.Second * 5)
if err = sendCmd("docker", "start", "clustertests_pilosa1_1"); err != nil {
t.Fatalf("sending start command: %v", err)
}
if err = sendCmd("docker", "start", "clustertests_pilosa2_1"); err != nil {
t.Fatalf("sending start command: %v", err)
}
if err = sendCmd("docker", "start", "clustertests_pilosa3_1"); err != nil {
t.Fatalf("sending start command: %v", err)
}
if err = backupCmd.Wait(); err == nil {
t.Fatal("backup command should have errored but didn't")
}
})
}
func waitForStatus(t *testing.T, stator func(context.Context) (string, error), status string, n int, sleep time.Duration) {

3
internal/clustertests/docker-compose.yml
View file

@ -9,6 +9,7 @@ services:
- "33455:10101"
environment:
- PILOSA_NAME=pilosa1
- PILOSA_ETCD_DIR=/root/.etcd
- PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
- PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa1:10201
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
@ -28,6 +29,7 @@ services:
- "33456:10101"
environment:
- PILOSA_NAME=pilosa2
- PILOSA_ETCD_DIR=/root/.etcd
- PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
- PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa2:10201
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
@ -47,6 +49,7 @@ services:
- "33457:10101"
environment:
- PILOSA_NAME=pilosa3
- PILOSA_ETCD_DIR=/root/.etcd
- PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
- PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa3:10201
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301

7
internal/clustertests/pause_node_test.go
View file

@ -23,11 +23,16 @@ import (
"github.com/pkg/errors"
)
func sendCmd(cmd string, args ...string) error {
func startCmd(cmd string, args ...string) (*exec.Cmd, error) {
pcmd := exec.Command(cmd, args...)
pcmd.Stdout = os.Stdout
pcmd.Stderr = os.Stderr
err := pcmd.Start()
return pcmd, err
}
func sendCmd(cmd string, args ...string) error {
pcmd, err := startCmd(cmd, args...)
if err != nil {
return errors.Wrap(err, "starting cmd")
}

12
lattice/src/services/__mocks__/eventServices.tsx Normal file
View file

@ -0,0 +1,12 @@
const pilosa = {
get: {
auth() {
return new Promise((resolve, reject) => {});
},
userinfo() {
return new Promise((resolve, reject) => {});
},
},
};
module.exports.pilosa = pilosa;

96
lattice/src/services/useAuth.test.tsx Normal file
View file

@ -0,0 +1,96 @@
import { AxiosResponse } from 'axios';
import { act } from 'react-dom/test-utils';
import ReactDOM from 'react-dom';
import { ProvideAuth, useAuth } from 'services/useAuth';
import { pilosa } from './eventServices';
jest.mock('./eventServices');
const AUTHENTICATED = 'Authenticated';
const NOTAUTHED = 'Not Authed';
const AUTHOFF = 'Auth off';
function TestUseAuthComponent() {
const auth = useAuth();
if (auth.isAuthOn === true && auth.isAuthenticated === true) {
return <div>{AUTHENTICATED}</div>;
} else if (auth.isAuthOn === true && auth.isAuthenticated === false) {
return <div>{NOTAUTHED}</div>;
} else {
return <div>{AUTHOFF}</div>;
}
}
beforeEach(() => {
jest.clearAllMocks();
});
test('test useAuth - expect authenticated', async () => {
const mockResponse: AxiosResponse<any> = {
status: 200,
data: 'OK',
statusText: '',
headers: {},
config: {},
};
const root = document.createElement('root');
await act(async () => {
jest.spyOn(pilosa.get, 'auth').mockResolvedValueOnce(mockResponse);
ReactDOM.render(
<ProvideAuth>
<TestUseAuthComponent />
</ProvideAuth>,
root
);
});
expect(pilosa.get.auth).toHaveBeenCalledTimes(1);
expect(root.innerHTML).toContain(AUTHENTICATED);
});
test('test useAuth - expect not authed', async () => {
const mockResponse: AxiosResponse<any> = {
status: 200,
data: '',
statusText: '',
headers: {},
config: {},
};
const root = document.createElement('root');
await act(async () => {
jest.spyOn(pilosa.get, 'auth').mockResolvedValueOnce(mockResponse);
ReactDOM.render(
<ProvideAuth>
<TestUseAuthComponent />
</ProvideAuth>,
root
);
});
expect(pilosa.get.auth).toHaveBeenCalledTimes(1);
expect(root.innerHTML).toContain(NOTAUTHED);
});
test('test useAuth - expect auth off', async () => {
const mockResponse: AxiosResponse<any> = {
status: 204,
data: '',
statusText: '',
headers: {},
config: {},
};
const root = document.createElement('root');
await act(async () => {
jest.spyOn(pilosa.get, 'auth').mockResolvedValueOnce(mockResponse);
ReactDOM.render(
<ProvideAuth>
<TestUseAuthComponent />
</ProvideAuth>,
root
);
});
expect(pilosa.get.auth).toHaveBeenCalledTimes(1);
expect(root.innerHTML).toContain(AUTHOFF);
});

0
qa/scripts/config.json Normal file
View file

7
qa/scripts/deployNode.sh
View file

@ -2,6 +2,9 @@
# To run script: ./deployNode.sh $PROFILE
# default to the VPC initially created
VPC=${VPC:-vpc-0582f594d7d2ca2d4}
function deploy_node() {
# get AMI, security group and subnet ID
AMI=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-ebs --query 'Parameters[0].[Value]' --output text --profile $PROFILE)
@ -10,13 +13,13 @@ function deploy_node() {
exit 1
fi
SECURITY_GROUP=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
SECURITY_GROUP=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text --profile $PROFILE)
if [[ $? > 0 ]]; then
echo "aws session manager failed to find security group"
exit 1
fi
SUBNET_ID=$(aws ec2 describe-subnets --filters 'Name=vpc-id,Values=vpc-03a4ba3d5b7c8f978' 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
SUBNET_ID=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" 'Name=availability-zone,Values=us-east-2a' --query 'Subnets[0].SubnetId' --output text --profile $PROFILE)
if [[ $? > 0 ]]; then
echo "aws session manager failed to find subnet ID"
exit 1

28
qa/scripts/deploySingleNodeCluster.sh Normal file
View file

@ -0,0 +1,28 @@
#!/bin/bash
# To run script: ./deploySingleNodeCluster.sh
# requires TF_VAR_gitlab_token env var to be set
echo “$(pwd)”
pushd ./qa/tf/ci/singlenode
export TF_IN_AUTOMATION=1
terraform init -input=false
terraform apply -input=false -auto-approve
popd
# configure Featurebase
# step 1a: get IPs of the cluster
# step 1b: get IPs of the ingest nodes
# step 2: write a featurebase.conf file
# step 3: write featurebase.service
# step 4: start featurebase
# step 5: verify featurebase running

17
qa/scripts/ingestWorkload.sh
View file

@ -1,7 +1,14 @@
#!/usr/bin/env bash
# path for featurebase binary
FEATUREBASE_PATH=/usr/local/bin
# path for directory with csv directory files for all fields to be ingested
CSV_DIR_PATH=/data
# To run:
# ./ingestWorkload.sh {Path for featurebase binary} {Local host & port for featurebase} {Path for directory with csv files} {initialize flag}
# ./ingestWorkload.sh {Local host & port for featurebase} {initialize flag}
function delete_field {
if (($INITIALIZE == 0));
@ -30,18 +37,10 @@ function ingest_set_field {
$FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE
}
# path for featurebase binary
FEATUREBASE_PATH=$1
shift
# featurebase host & port
HOST=$1
shift
# path for directory with csv directory files for all fields to be ingested
CSV_DIR_PATH=$1
shift
# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
INITIALIZE=$1
shift

3
qa/scripts/perf.sh Normal file
View file

@ -0,0 +1,3 @@
#!/bin/bash
echo >&2 "performance testing"
time ./simulacraData

8
qa/scripts/runSamsungGauntlet.sh Normal file
View file

@ -0,0 +1,8 @@
#!/bin/bash
#openssl rand -base64 32 | tr -d /=+ | cut -c -16
./setupSamsungGauntlet.sh
./testSamsungGauntlet.sh
./teardownSamsungGauntlet.sh

41
qa/scripts/setupSamsungGauntlet.sh Executable file
View file

@ -0,0 +1,41 @@
#!/bin/bash
# To run script: ./setupSamsungGauntlet.sh
# requires TF_VAR_gitlab_token env var to be set
pushd ./qa/tf/gauntlet/samsung
export TF_IN_AUTOMATION=1
echo "Running terraform init..."
terraform init -input=false
echo "Running terraform apply..."
terraform apply -input=false -auto-approve
terraform output -json > samsung-gauntlet.json
popd
# get the bastion host
BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
echo "using bastion ${BASTION}"
NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
echo "using node ${NODE}"
# remember that the nodes will take at least 2 mins to be up and going and finish cloud-init
#while true
#do
# nc -G 2 -w 1 $BASTION 22
# if [ $? -eq 0 ]
# then
# break
# fi
#done
sleep 150
# verify featurebase running
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${BASTION} "curl -s http://${NODE}:10101/status"
if (( $? != 0 ))
then
echo "Featurebase cluster not running"
exit 1
fi

8
qa/scripts/teardownSamsungGauntlet.sh Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
# To run script: ./teardownSamsungGauntlet.sh
# requires TF_VAR_gitlab_token env var to be set
cd qa/tf/gauntlet/samsung
export TF_IN_AUTOMATION=1
terraform destroy -auto-approve

48
qa/scripts/testSamsungGauntlet.sh Executable file
View file

@ -0,0 +1,48 @@
#!/bin/bash
# get the bastion host
BASTION=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.ingest_ips][0]["value"][0]')
echo "using bastion ${BASTION}"
NODE=$(cat ./qa/tf/gauntlet/samsung/samsung-gauntlet.json | jq -r '[.data_node_ips][0]["value"][0]')
echo "using node ${NODE}"
# generate csv files
GOOS=linux GOARCH=arm64 go build ./qa/simulacraData/...
scp -i ~/.ssh/gitlab-featurebase-ci.pem simulacraData ec2-user@${BASTION}:/data
if (( $? != 0 ))
then
echo "Copy failed"
exit 1
fi
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "cd /data && /data/simulacraData"
if (( $? != 0 ))
then
echo "Making big files failed"
exit 1
fi
# ingest these files the way that samsung does it
scp -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/testSamsungPayload.sh ec2-user@${BASTION}:
if (( $? != 0 ))
then
echo "Copy ingest script failed"
exit 1
fi
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 1"
if (( $? != 0 ))
then
echo "Running 1 testSamsungPayload.sh failed"
exit 1
fi
ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@${BASTION} "./testSamsungPayload.sh http://${NODE}:10101 0"
if (( $? != 0 ))
then
echo "Running 0 testSamsungPayload.sh failed"
exit 1
fi
# query workload that runs

84
qa/scripts/testSamsungPayload.sh Executable file
View file

@ -0,0 +1,84 @@
#!/usr/bin/env bash
# path for featurebase binary
FEATUREBASE_PATH=/usr/local/bin
# path for directory with csv directory files for all fields to be ingested
CSV_DIR_PATH=/data
# To run:
# ./testSamsungPayload.sh {Local host & port for featurebase} {initialize flag}
function delete_field {
if (($INITIALIZE == 0));
then
curl -XDELETE $HOST/index/$INDEX/field/$FIELD
fi
}
# Script to replicate samsung workload of deleting and re-ingesting fields every night
# outline delete and re-ingest workload
function ingest_int_field {
delete_field
curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"type": "int", "min": 0, "max":'$MAX'}}'
$FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE
}
function ingest_time_field {
delete_field
curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true, "type": "time", "timeQuantum": "YMD"}}'
$FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE
}
function ingest_set_field {
delete_field
curl -XPOST $HOST/index/$INDEX/field/$FIELD -d '{"options": {"keys": true}}'
$FEATUREBASE_PATH/featurebase import --host $HOST -i $INDEX -f $FIELD $CSV_FILE
}
# featurebase host & port
HOST=$1
shift
# intialize flag - 0:disabled, 1:enabled - creates the index and fields for testing
INITIALIZE=$1
shift
# get a list of csv files in the directory
CSV_FILES=`ls $CSV_DIR_PATH/*.csv`
# assign index name
INDEX="samsung"
if (($INITIALIZE == 1));
then
curl -XPOST $HOST/index/$INDEX
fi
# perform delete and re-ingest for all fields
for CSV_FILE in ${CSV_FILES[@]}
do
# get field name from csv file path
FIELD="$(basename $CSV_FILE .csv)"
if [[ "$FIELD" == *"age"* ]];
then
MAX=100
ingest_int_field
elif [[ "$FIELD" == *"identifier"* ]];
then
MAX=$((2**63 - 1)) # compute max value for 64bit
ingest_int_field
elif [[ "$FIELD" == *"ip"* ]];
then
MAX=$((2**31 - 1)) # compute max value for 32bit
ingest_int_field
elif [[ "$FIELD" == *"time"* ]];
then
ingest_time_field
else
ingest_set_field
fi
done

37
qa/tf/.modules/featurebase-cluster/README.md Normal file
View file

@ -0,0 +1,37 @@
# Summary
This module provisions a VPC, subnets, instances, keys, and security groups needed for a basic featurebase cluster running in AWS. It is meant to be used as a module. For example:
```hcl
module "featurebase" {
source "/path/to/module/"
cluster_prefix = "sprockets"
azs = ["us-east-1a", "us-east-1b", "us-east-1c"]
}
```
The path to the module is wherever the `featurebase-cloud` directory is. So if you have put it in `/var/opt/terraform/modules/featurebase-cloud` then calling the module would look like:
```hcl
module "featurebase" {
source "/var/opt/terraform/modules/featurebase-cloud"
cluster_prefix = "sprockets"
}
```
Much more is configurable; for a complete list, look in `variables.tf`. Reasonable defaults have been set.
## AWS Access
Please make sure you have set up your AWS access in either environment variables, or in the credentials file.
Some useful links for this are:
AWS Environment Variables <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html>
## State
State is currently kept locally, for as this is intended for PoCs. It can be stored in a remote s3 or GCS bucket if desired.

229
qa/tf/.modules/featurebase-cluster/main.tf Normal file
View file

@ -0,0 +1,229 @@
data "aws_ami" "amazon_linux_2" {
most_recent = true
owners = ["amazon"]
filter {
name = "name"
values = ["amzn2-ami-hvm-*"]
}
filter {
name = "virtualization-type"
values = ["hvm"]
}
filter {
name = "architecture"
values = ["arm64"]
}
}
resource "aws_instance" "fb_cluster_nodes" {
count = var.fb_data_node_count
ami = data.aws_ami.amazon_linux_2.id
instance_type = var.fb_data_node_type
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
vpc_security_group_ids = [aws_security_group.featurebase.id]
monitoring = true
subnet_id = var.subnet != "" ? var.subnet : module.vpc.private_subnets[count.index % length(module.vpc.private_subnets)]
availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
root_block_device {
volume_type = "gp3"
volume_size = 20
}
ebs_block_device {
device_name = "/dev/sdb"
volume_type = var.fb_data_disk_type
volume_size = var.fb_data_disk_size_gb
iops = var.fb_data_disk_iops
}
tags = {
Prefix = "${var.cluster_prefix}"
Name = "${var.cluster_prefix}-featurebase-cluster-${count.index}"
Role = "cluster_node"
}
user_data = base64encode(templatefile("${path.module}/setup_cluster_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_data_node_count, fb_cluster_replica_count = var.fb_cluster_replica_count, region = var.region }))
}
resource "aws_instance" "fb_ingest" {
count = var.fb_ingest_node_count
ami = data.aws_ami.amazon_linux_2.id
key_name = aws_key_pair.gitlab-featurebase-ci.key_name
vpc_security_group_ids = [aws_security_group.ingest.id]
instance_type = var.fb_ingest_type
associate_public_ip_address = true
monitoring = true
subnet_id = var.subnet != "" ? var.subnet : module.vpc.public_subnets[count.index % length(module.vpc.public_subnets)]
availability_zone = var.zone != "" ? var.zone : var.azs[count.index % length(var.azs)]
iam_instance_profile = "${aws_iam_instance_profile.fb_cluster_node_profile.name}"
root_block_device {
volume_type = "gp3"
volume_size = 20
}
ebs_block_device {
device_name = "/dev/sdb"
volume_type = var.fb_ingest_disk_type
volume_size = var.fb_ingest_disk_size_gb
iops = var.fb_ingest_disk_iops
}
tags = {
Prefix = "${var.cluster_prefix}"
Name = "${var.cluster_prefix}-featurebase-ingest-${count.index}"
Role = "ingest_node"
}
user_data = base64encode(templatefile("${path.module}/setup_ingest_node.sh.tpl", { gitlab_token = var.gitlab_token, cluster_prefix = var.cluster_prefix, node_count = var.fb_ingest_node_count, this_node = count.index, region = var.region }))
}
resource "aws_key_pair" "gitlab-featurebase-ci" {
key_name = "gitlab-featurebase-ci"
public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL"
}
resource "aws_security_group" "featurebase" {
name = "allow_featurebase"
description = "Allow featurebase inbound traffic"
vpc_id = module.vpc.vpc_id
ingress {
description = "TLS from Internal"
from_port = 10101
to_port = 10101
protocol = "tcp"
cidr_blocks = [module.vpc.vpc_cidr_block]
}
ingress {
description = "GRPC from Internal"
from_port = 20101
to_port = 20101
protocol = "tcp"
cidr_blocks = [module.vpc.vpc_cidr_block]
}
ingress {
description = "PostgreSQL from Internal"
from_port = 55432
to_port = 55432
protocol = "tcp"
cidr_blocks = [module.vpc.vpc_cidr_block]
}
ingress {
description = "etcd from internal"
from_port = 10301
to_port = 10301
protocol = "tcp"
cidr_blocks = [module.vpc.vpc_cidr_block]
}
ingress {
description = "etcd from internal 2"
from_port = 10401
to_port = 10401
protocol = "tcp"
cidr_blocks = [module.vpc.vpc_cidr_block]
}
ingress {
description = "SSH"
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
tags = {
Name = "allow_featurebase"
}
}
resource "aws_security_group" "ingest" {
name = "allow_ingest"
description = "Allow ingest inbound traffic"
vpc_id = module.vpc.vpc_id
ingress {
from_port = 10101
to_port = 10101
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
ingress {
description = "SSH"
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
ipv6_cidr_blocks = ["::/0"]
}
tags = {
Name = "allow_ingest"
}
}
resource "aws_iam_instance_profile" "fb_cluster_node_profile" {
name = "fb_cluster_node_profile"
role = aws_iam_role.fb_cluster_node_role.name
}
resource "aws_iam_role" "fb_cluster_node_role" {
name = "fb_cluster_node"
assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = "sts:AssumeRole"
Effect = "Allow"
Sid = ""
Principal = {
Service = "ec2.amazonaws.com"
}
},
]
})
inline_policy {
name = "ec2_read_all"
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = ["ec2:Describe*"]
Effect = "Allow"
Resource = "*"
},
]
})
}
}

7
qa/tf/.modules/featurebase-cluster/outputs.tf Normal file
View file

@ -0,0 +1,7 @@
output "ingest_ips" {
value = aws_instance.fb_ingest.*.public_ip
}
output "data_node_ips" {
value = aws_instance.fb_cluster_nodes.*.private_ip
}

11
qa/tf/.modules/featurebase-cluster/provider.tf Normal file
View file

@ -0,0 +1,11 @@
terraform {
required_version = ">= 0.13.1"
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 3.38.0"
}
}
}

188
qa/tf/.modules/featurebase-cluster/setup_cluster_node.sh.tpl Normal file
View file

@ -0,0 +1,188 @@
#!/bin/bash
#path to the featurebase.conf file
CONFIG_FILE_PATH="/etc/featurebase.conf"
#path to the featurebase.service file
SERVICE_FILE_PATH="/etc/systemd/system/featurebase.service"
AWS_INSTANCE_ID=""
#IP of this node
PRIVATE_IP=""
PRIVATE_IP_INDEX=-1
#IPs of the cluster
CLUSTER_IPS=""
get_aws_instance_id() {
echo "Getting AWS instance ID..."
while true
do
curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null
if [ $? -eq 0 ]
then
break
fi
done
AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`
echo "AWS instance ID is: $${AWS_INSTANCE_ID}"
}
wait_on_all_cluster_ips() {
echo "Waiting on all cluster IPs..."
# get IP for node
IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
IP_LENGTH=`echo "$IPS" | wc -l`
for i in {0..24}
do
echo "Comparing $${IP_LENGTH} with ${node_count}"
if [ $IP_LENGTH == "${node_count}" ]; then
echo "Cluster is up after $${i} tries."
break
fi
sleep 10s
done
if [ $IP_LENGTH != "${node_count}" ]; then
echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes."
exit 1
fi
}
get_private_ip() {
echo "Getting private IP address..."
PRIVATE_IP=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=instance-id,Values=$${AWS_INSTANCE_ID}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
echo "Private IP is $${PRIVATE_IP}"
}
get_cluster_ips() {
echo "Getting cluster IPs..."
# get IP for node
IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=cluster_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
IP_LENGTH=`echo "$IPS" | wc -l`
IFS=$'\n'
cnt=0
for ip in $IPS
do
echo $cnt $ip
if (($cnt + 1 != $IP_LENGTH))
then
CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301,"
else
CLUSTER_IPS="$${CLUSTER_IPS}p$${cnt}=http://$ip:10301"
fi
echo "comparing $ip to $PRIVATE_IP"
if [ "$ip" = "$PRIVATE_IP" ]; then
PRIVATE_IP_INDEX=$cnt
fi
cnt=$((cnt+1))
done
echo "CLUSTER_IPS are: $${CLUSTER_IPS}"
}
write_featurebase_config_file() {
echo "Writing featurebase.conf file..."
cat << EOT > $${CONFIG_FILE_PATH}
name = "p$${PRIVATE_IP_INDEX}"
bind = "0.0.0.0:10101"
bind-grpc = "0.0.0.0:20101"
data-dir = "/data/featurebase"
log-path = "/var/log/molecula/featurebase.log"
max-file-count=900000
max-map-count=900000
long-query-time = "10s"
[postgres]
bind = "localhost:55432"
[cluster]
name = "${cluster_prefix}"
replicas = ${fb_cluster_replica_count}
[etcd]
listen-client-address = "http://$${PRIVATE_IP}:10401"
listen-peer-address = "http://$${PRIVATE_IP}:10301"
initial-cluster = "$${CLUSTER_IPS}"
[metric]
service = "prometheus"
EOT
echo "featurebase.conf written to $${CONFIG_FILE_PATH}."
}
write_featurebase_service_file() {
echo "Writing featurebase.service file..."
cat << EOT > $${SERVICE_FILE_PATH}
# Not Ansible managed
[Unit]
Description="Service for FeatureBase"
[Service]
RestartSec=30
Restart=on-failure
EnvironmentFile=
User=molecula
ExecStart=/usr/local/bin/featurebase server -c /etc/featurebase.conf
[Install]
EOT
echo "featurebase.service written to $${SERVICE_FILE_PATH}."
}
#get the instance id
get_aws_instance_id
#copy the script so we can look at it later if needed
sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt /home/ec2-user/setup_cluster_node.sh
#wait for the count of nodes to equal requested nodes
wait_on_all_cluster_ips
#get private ip
get_private_ip
#generate cluster ips
get_cluster_ips
#write the featurebase config file
write_featurebase_config_file
#write the featurebase service file
write_featurebase_service_file
#get the featurebase binary and put in in the right spot
echo "Getting featurebase binary..."
curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"
mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase
echo "featurebase binary copied."
sudo mkdir /data
sudo mkfs.ext4 /dev/nvme1n1
sudo mount /dev/nvme1n1 /data
adduser molecula
sudo mkdir /var/log/molecula
sudo chown molecula /var/log/molecula
sudo mkdir -p /data/featurebase
sudo chown molecula /data/featurebase
sudo systemctl daemon-reload
sudo systemctl start featurebase
sudo systemctl enable featurebase
sudo systemctl status featurebase
echo "Done!"

66
qa/tf/.modules/featurebase-cluster/setup_ingest_node.sh.tpl Normal file
View file

@ -0,0 +1,66 @@
#!/bin/bash
AWS_INSTANCE_ID=""
get_aws_instance_id() {
echo "Getting AWS instance ID..."
while true
do
curl -s http://169.254.169.254/latest/meta-data/instance-id > /dev/null
if [ $? -eq 0 ]
then
break
fi
done
AWS_INSTANCE_ID=`curl http://169.254.169.254/latest/meta-data/instance-id`
echo "AWS instance ID is: $${AWS_INSTANCE_ID}"
}
wait_on_all_ingest_ips() {
echo "Waiting on all cluster IPs..."
# get IP for node
IPS=$(aws ec2 describe-instances --filters "Name=instance-state-name, Values=running" "Name=tag:Role, Values=ingest_node" "Name=tag:Prefix, Values=${cluster_prefix}" --query 'Reservations[*].Instances[*].PrivateIpAddress' --output text --region ${region})
IP_LENGTH=`echo "$IPS" | wc -l`
for i in {0..24}
do
echo "Comparing $${IP_LENGTH} with ${node_count}"
if [ $IP_LENGTH == "${node_count}" ]; then
echo "Cluster is up after $${i} tries."
break
fi
sleep 10s
done
if [ $IP_LENGTH != "${node_count}" ]; then
echo "Timed out waiting for cluster to be available $${IP_LENGTH} actual nodes compared with ${node_count} desire nodes."
exit 1
fi
}
#copy the script so we can look at it later if needed
sudo cp /var/lib/cloud/instances/$${AWS_INSTANCE_ID}/user-data.txt ~/setup_ingest_node.sh
#get the instance id
get_aws_instance_id
#wait for the count of nodes to equal requested nodes
wait_on_all_ingest_ips
echo "Getting featurebase binary..."
curl --header "PRIVATE-TOKEN: ${gitlab_token}" -o "/home/ec2-user/featurebase_linux_arm64" https://gitlab.com/api/v4/projects/molecula%2Ffeaturebase/jobs/artifacts/master/raw/featurebase_linux_arm64?job=build%20for%20linux%20arm64
chown ec2-user:ec2-user "/home/ec2-user/featurebase_linux_arm64"
chmod ugo+x "/home/ec2-user/featurebase_linux_arm64"
mv /home/ec2-user/featurebase_linux_arm64 /usr/local/bin/featurebase
echo "featurebase binary copied."
sudo mkdir /data
sudo mkfs.ext4 /dev/nvme1n1
sudo mount /dev/nvme1n1 /data
sudo chown -R ec2-user /data

93
qa/tf/.modules/featurebase-cluster/variables.tf Normal file
View file

@ -0,0 +1,93 @@
variable "cluster_prefix" {
type = string
description = "This is a identifier that will be prefixed to created resources"
}
variable "fb_ingest_type" {
type = string
default = "c6g.2xlarge"
}
variable "fb_ingest_node_count" {
type = number
default = 1
}
variable "fb_data_node_type" {
type = string
default = "c6g.16xlarge"
}
variable "fb_data_node_count" {
type = number
default = 3
}
variable "fb_cluster_replica_count" {
type = number
default = 1
}
variable "subnet" {
default = ""
}
variable "zone" {
default = ""
}
variable "fb_data_disk_type" {
default = "gp3"
}
variable "fb_data_disk_iops" {
default = 1000
}
variable "fb_data_disk_size_gb" {
default = 100
}
variable "fb_ingest_disk_type" {
default = "gp3"
}
variable "fb_ingest_disk_iops" {
default = 1000
}
variable "fb_ingest_disk_size_gb" {
default = 100
}
variable "azs" {
type = list(any)
default = ["us-east-2a", "us-east-2b", "us-east-2c"]
}
variable "private_subnets" {
type = list(any)
default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
}
variable "public_subnets" {
type = list(any)
default = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"]
}
variable "vpc_cidr" {
default = "10.0.0.0/16"
}
variable "region" {
description = "Region to create AWS resources in"
type = string
}
variable "profile" {
description = "Profile to use to authenticate with AWS"
type = string
}
variable "gitlab_token" {
description = "Gitlab API token"
type = string
}

16
qa/tf/.modules/featurebase-cluster/vpc.tf Normal file
View file

@ -0,0 +1,16 @@
module "vpc" {
source = "terraform-aws-modules/vpc/aws"
name = "${var.cluster_prefix}"
cidr = var.vpc_cidr
azs = var.azs
private_subnets = var.private_subnets
public_subnets = var.public_subnets
enable_nat_gateway = true
enable_vpn_gateway = false
tags = {
Name = "${var.cluster_prefix}"
}
}

16
qa/tf/README.md Normal file
View file

@ -0,0 +1,16 @@
# Deploy testing environments with this one weird trick!
This directory contains Terraform to deploy test environments both ad-hoc and as part of CI/CD pipelines.
The .modules contains the guts of the operation, the things you probably want are in the other directories, each with a README.
## How to Terraform
With terraform installed (`brew install terraform` if not)...
You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
## Other prerequisites:
Please read these carefully.

10
qa/tf/ci/singlenode/main.tf Normal file
View file

@ -0,0 +1,10 @@
module "ci-cluster" {
source = "../../.modules/featurebase-cluster"
cluster_prefix = "ci-single-node"
region = var.region
profile = var.profile
fb_data_node_type = "m6g.large"
fb_data_node_count = 1
gitlab_token = var.gitlab_token
}

9
qa/tf/ci/singlenode/outputs.tf Normal file
View file

@ -0,0 +1,9 @@
output "ingest_ips" {
description = "List of ingest IPs"
value = module.ci-cluster.ingest_ips
}
output "data_node_ips" {
description = "List of data node IPs"
value = module.ci-cluster.data_node_ips
}

4
qa/tf/ci/singlenode/provider.tf Normal file
View file

@ -0,0 +1,4 @@
provider "aws" {
region = var.region
profile = var.profile
}

2
qa/tf/ci/singlenode/tf.auto.tfvars Normal file
View file

@ -0,0 +1,2 @@
region = "us-east-2"
profile = "service-terraform"

14
qa/tf/ci/singlenode/variables.tf Normal file
View file

@ -0,0 +1,14 @@
variable "region" {
description = "The AWS region in which the VPC should be built"
type = string
}
variable "profile" {
description = "The name of the AWS profile Terraform should use for auth."
type = string
}
variable "gitlab_token" {
description = "The API token for taking to Gitlab API - expected to come from an env variable."
type = string
}

35
qa/tf/gauntlet/samsung/README.md Normal file
View file

@ -0,0 +1,35 @@
With terraform installed (`brew install terraform` if not)...
You can do `terraform plan` -> `terraform apply` to spin up a cluster, `terraform destroy` to tear one down.
## Other prerequisites:
Please read these carefully.
Be in the `tf` directory (e.g., when you try to run a `terraform` command, the output of `pwd` should be `.../featurebase/qa/tf`)
Currently, the path to the terraform module is using a local reference, i.e., in `main.tf`, the source line is assuming that you have `molecula-terraform` project installed locally, such that the `molecular-terraform` project and `featurebase` have the same parent directory (e.g., `...A/featurebase/qa/tf` and `...A/molecular-terraform/aws/.modules/featurebase-cluster` should both be valid paths).
In addition, you must currently have a local copy of the `fb901` branch for the `molecular-terraform` project (located in the previously specified directory).
Last thing, there is a key that is currently in 1Password (in the `Shared` vault, called `gitlab-featurebase-ci AWS key`) that must be in `~/.ssh/`, `chmod 400`, named `gitlab-featurebase-ci.pem`. You need this key to SSH to these instances. Assuming an `~/.ssh/config` like the following (append to the top of yours)
```
Host test_*
User ec2-user
IdentityFile ~/.ssh/gitlab-featurebase-ci.pem
Host test_ingest
HostName 3.143.237.165
Host test_node
HostName 10.0.1.142
ProxyJump test_ingest
```
except with the `test_ingest`'s `HostName` being the public, `ingest_ips` output from `terraform output` and `test_node`'s `HostName` being one of the private, `data_node_ips` output from `terraform output`. (Hopefully the rationale to use the ssh config to do the jumping like this makes sense; you can do `ssh test_ingest` or `ssh test_node` with minimal further fiddling.)
OR specify cert to us directly thus:
`ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem ec2-user@ip_address`
-A is used to ensure key forwarding.
### TODOs
* We need a `user-data.sh` script which sets up/installs featurebase (possibly installs go, most likely pulls the artifacts from GitLab; sets up featurebase on both the node and data workers).
* Logs get sent to DataDog?

13
qa/tf/gauntlet/samsung/main.tf Normal file
View file

@ -0,0 +1,13 @@
module "samsung-cluster" {
source = "../../.modules/featurebase-cluster"
cluster_prefix = "samsung-gauntlet"
region = var.region
profile = var.profile
fb_data_node_type = "m6g.xlarge"
fb_data_disk_iops = 10000
fb_data_node_count = 3
fb_ingest_type = "m6g.large"
fb_ingest_disk_iops = 10000
fb_ingest_node_count = 1
gitlab_token = var.gitlab_token
}

9
qa/tf/gauntlet/samsung/outputs.tf Normal file
View file

@ -0,0 +1,9 @@
output "ingest_ips" {
description = "List of ingest IPs"
value = module.samsung-cluster.ingest_ips
}
output "data_node_ips" {
description = "List of data node IPs"
value = module.samsung-cluster.data_node_ips
}

4
qa/tf/gauntlet/samsung/provider.tf Normal file
View file

@ -0,0 +1,4 @@
provider "aws" {
region = var.region
profile = var.profile
}

30
qa/tf/gauntlet/samsung/samsung-gauntlet.json Normal file
View file

@ -0,0 +1,30 @@
{
"data_node_ips": {
"sensitive": false,
"type": [
"tuple",
[
"string",
"string",
"string"
]
],
"value": [
"10.0.1.144",
"10.0.2.108",
"10.0.3.178"
]
},
"ingest_ips": {
"sensitive": false,
"type": [
"tuple",
[
"string"
]
],
"value": [
"3.145.104.76"
]
}
}

2
qa/tf/gauntlet/samsung/tf.auto.tfvars Normal file
View file

@ -0,0 +1,2 @@
region = "us-east-2"
profile = "service-terraform"

14
qa/tf/gauntlet/samsung/variables.tf Normal file
View file

@ -0,0 +1,14 @@
variable "region" {
description = "The AWS region in which the VPC should be built"
type = string
}
variable "profile" {
description = "The name of the AWS profile Terraform should use for auth."
type = string
}
variable "gitlab_token" {
description = "The API token for taking to Gitlab API - expected to come from an env variable."
type = string
}

4
rbf/cursor.go
View file

@ -932,6 +932,10 @@ func (c *Cursor) First() error {
case PageTypeBranch:
elem.index = 0
if n := readCellN(buf); elem.index >= n { // branch cell index must less than cell count
return fmt.Errorf("branch cell index out of range: pgno=%d i=%d n=%d", elem.pgno, elem.index, n)
}
// Read cell pgno into the next stack level.
cell := readBranchCell(buf, elem.index)

17
rbf/db.go
View file

@ -7,6 +7,8 @@ import (
"io"
"os"
"path/filepath"
"runtime/debug"
"sort"
"sync"
"syscall"
@ -637,6 +639,7 @@ func (db *DB) Begin(writable bool) (_ *Tx, err error) {
pageMap: db.pageMap,
walPageN: db.walPageN,
writable: writable,
stack: debug.Stack(), // DEBUG
DeleteEmptyContainer: true,
}
@ -815,6 +818,20 @@ func (db *DB) getCursor(tx *Tx) *Cursor {
return c
}
func (db *DB) DebugInfo() *DebugInfo {
info := &DebugInfo{Path: db.Path}
for tx := range db.txs {
info.Txs = append(info.Txs, tx.DebugInfo())
}
sort.Slice(info.Txs, func(i, j int) bool { return info.Txs[i].Ptr < info.Txs[j].Ptr })
return info
}
type DebugInfo struct {
Path string `json:"path"`
Txs []*TxDebugInfo `json:"txs"`
}
// Shared pool for in-memory database pages.
// These are used before being flushed to disk.
var pagePool = &sync.Pool{

15
rbf/db_test.go
View file

@ -339,6 +339,21 @@ func TestDB_MultiTx(t *testing.T) {
}
}
func TestDB_DebugInfo(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer tx.Rollback()
info := db.DebugInfo()
if got, want := info.Path, db.Path; got != want {
t.Fatalf("Path=%q, want %q", got, want)
} else if got, want := len(info.Txs), 1; got != want {
t.Fatalf("len(Txs)=%d, want %d", got, want)
}
}
// premake pool of random values
const randPool = (1 << 18)

43
rbf/rbf.go
View file

@ -799,3 +799,46 @@ func (m *Metric) Inc(d time.Duration) {
fmt.Printf("metric:%10s avg=%dns\n", m.name, int(m.d)/m.n)
}
}
// ErrorList represents a list of errors.
type ErrorList []error
// Err returns the list if it contains errors. Otherwise returns nil.
func (a ErrorList) Err() error {
if len(a) > 0 {
return a
}
return nil
}
func (a ErrorList) Error() string {
switch len(a) {
case 0:
return "no errors"
case 1:
return a[0].Error()
}
return fmt.Sprintf("%s (and %d more errors)", a[0], len(a)-1)
}
func (a ErrorList) FullError() string {
if len(a) == 0 {
return ""
}
var buf bytes.Buffer
for _, err := range a {
fmt.Fprintln(&buf, err)
}
return buf.String()
}
// Append appends an error to the list. If err is an ErrorList then all errors are appended.
func (a *ErrorList) Append(err error) {
switch err := err.(type) {
case ErrorList:
*a = append(*a, err...)
default:
*a = append(*a, err)
}
}

BIN
rbf/rbf/testdata/check/bad-freelist/data vendored Normal file
View file

Binary file not shown.

0
rbf/rbf/testdata/check/bad-freelist/wal vendored Normal file
View file

19
rbf/rbf_test.go
View file

@ -54,17 +54,30 @@ func NewDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB {
if err != nil {
panic(err)
}
return NewDBAt(tb, path, cfg...)
}
// NewDBAt returns a new instance of DB with a given path.
func NewDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB {
var cfg0 *rbfcfg.Config
if len(cfg) > 0 {
cfg0 = cfg[0]
}
db := rbf.NewDB(path, cfg0)
return db
return rbf.NewDB(path, cfg0)
}
// MustOpenDB returns a db opened on a temporary file. On error, fail test.
func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB {
tb.Helper()
path, err := testhook.TempDir(tb, "rbfdb")
if err != nil {
panic(err)
}
return MustOpenDBAt(tb, path, cfg...)
}
// MustOpenDBAt returns a db opened on an existing file. On error, fail test.
func MustOpenDBAt(tb testing.TB, path string, cfg ...*rbfcfg.Config) *rbf.DB {
tb.Helper()
if len(cfg) == 0 || cfg[0] == nil {
newconf := rbfcfg.NewDefaultConfig()
@ -73,7 +86,7 @@ func MustOpenDB(tb testing.TB, cfg ...*rbfcfg.Config) *rbf.DB {
} else if cfg[0].Logger == nil {
cfg[0].Logger = logger.NewLogfLogger(tb)
}
db := NewDB(tb, cfg...)
db := NewDBAt(tb, path, cfg...)
if err := db.Open(); err != nil {
tb.Fatal(err)
}

BIN
rbf/testdata/check/bad-bitmap/data vendored Normal file
View file

Binary file not shown.

0
rbf/testdata/check/bad-bitmap/wal vendored Normal file
View file

BIN
rbf/testdata/check/bad-freelist/data vendored Normal file
View file

Binary file not shown.

0
rbf/testdata/check/bad-freelist/wal vendored Normal file
View file

185
rbf/tx.go
View file

@ -65,6 +65,9 @@ type Tx struct {
// manages to trigger a *deallocation* (which I don't think should be
// happening), we'll process that one after the current list is processed.
pendingFreelistAdds []uint32
// DEBUG
stack []byte
}
func (tx *Tx) DBPath() string {
@ -735,10 +738,11 @@ func (tx *Tx) Check() error {
return ErrTxClosed
}
var errorList ErrorList
if err := tx.checkPageAllocations(); err != nil {
return fmt.Errorf("page allocations: %w", err)
errorList.Append(err)
}
return nil
return errorList.Err()
}
func (tx *Tx) checkPage(pgno, parent, typ uint32) error {
@ -764,14 +768,15 @@ func (tx *Tx) checkBranchPage(pgno, parent, typ uint32) error {
// checkPageAllocations ensures that all pages are either in-use or on the freelist.
func (tx *Tx) checkPageAllocations() error {
var errorList ErrorList
freePageSet, err := tx.freePageSet()
if err != nil {
return err
errorList.Append(err)
}
inusePageSet, err := tx.inusePageSet()
if err != nil {
return err
errorList.Append(err)
}
// Iterate over all pages and ensure they are either in-use or free.
@ -782,26 +787,23 @@ func (tx *Tx) checkPageAllocations() error {
_, isFree := freePageSet[pgno]
if isInuse && isFree {
return fmt.Errorf("page in-use & free: pgno=%d", pgno)
} else if !isInuse && !isFree {
page, _, err := tx.readPage(pgno)
if err != nil {
return err
}
flags := readFlags(page)
if flags == PageTypeBranch || flags == PageTypeLeaf {
return fmt.Errorf("page not in-use & not free: pgno=%d", pgno)
}
//assuming its a bitmap so its ok TODO ben?
return nil
errorList.Append(fmt.Errorf("page in-use & free: pgno=%d", pgno))
continue
}
if !isInuse && !isFree {
errorList.Append(fmt.Errorf("page not in-use & not free: pgno=%d", pgno))
continue
}
}
return nil
return errorList.Err()
}
// freePageSet returns the set of pages in the freelist.
func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {
var errorList ErrorList
m := make(map[uint32]struct{})
c := Cursor{tx: tx}
c.stack.elems[0] = stackElem{pgno: readMetaFreelistPageNo(tx.meta[:])}
@ -813,18 +815,20 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {
for {
if err := c.Next(); err == io.EOF {
return m, nil
return m, errorList.Err()
} else if err != nil {
return m, err
errorList.Append(err)
return m, errorList.Err()
}
elem := &c.stack.elems[c.stack.top]
leafPage, _, err := c.tx.readPage(elem.pgno)
if err != nil {
return nil, err
errorList.Append(fmt.Errorf("cannot read free page: pgno=%d err=%w", elem.pgno, err))
continue
}
cell := readLeafCell(leafPage, elem.index)
cell := readLeafCell(leafPage, elem.index)
for _, v := range cell.Values(tx) {
pgno := uint32((cell.Key << 16) | uint64(v))
m[pgno] = struct{}{}
@ -834,6 +838,7 @@ func (tx *Tx) freePageSet() (map[uint32]struct{}, error) {
// inusePageSet returns the set of pages in use by the root records or b-trees.
func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
var errorList ErrorList
m := make(map[uint32]struct{})
m[0] = struct{}{} // meta page
@ -843,15 +848,24 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
page, _, err := tx.readPage(pgno)
if err != nil {
return nil, err
errorList.Append(err)
break
}
pgno = WalkRootRecordPages(page)
}
// Traverse freelist and mark pages as in-use.
if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32) error {
if err := tx.walkTree(readMetaFreelistPageNo(tx.meta[:]), 0, func(pgno, parent, typ uint32, err error) error {
if err != nil {
errorList.Append(err)
return nil
}
m[pgno] = struct{}{}
return tx.checkPage(pgno, parent, typ)
if err := tx.checkPage(pgno, parent, typ); err != nil {
errorList.Append(err)
}
return nil
}); err != nil {
return m, err
}
@ -859,22 +873,28 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
// Traverse every b-tree and mark pages as in-use.
records, err := tx.RootRecords()
if err != nil {
return m, err
}
errorList.Append(err)
} else {
for itr := records.Iterator(); !itr.Done(); {
_, pgno := itr.Next()
for itr := records.Iterator(); !itr.Done(); {
_, pgno := itr.Next()
if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error {
if err != nil {
errorList.Append(err)
}
if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error {
m[pgno] = struct{}{}
return tx.checkPage(pgno, parent, typ)
}); err != nil {
return m, err
m[pgno] = struct{}{}
if err := tx.checkPage(pgno, parent, typ); err != nil {
errorList.Append(err)
}
return nil
}); err != nil {
return m, err
}
}
}
return m, nil
return m, errorList.Err()
}
// GetSizeBytesWithPrefix returns the size of bitmaps with a given key prefix.
@ -894,9 +914,9 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) {
}
// Traverse the bitmap's b-tree and count the bytes for each page.
if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32) error {
if err := tx.walkTree(pgno.(uint32), 0, func(pgno, parent, typ uint32, err error) error {
n += PageSize
return nil
return err
}); err != nil {
return 0, err
}
@ -905,21 +925,19 @@ func (tx *Tx) GetSizeBytesWithPrefix(prefix string) (n uint64, err error) {
}
// walkTree recursively iterates over a page and all its children.
func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) error) error {
func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32, err error) error) error {
// Read page and iterate over children.
page, _, err := tx.readPage(pgno)
if err != nil {
return err
return fn(pgno, parent, 0, fmt.Errorf("cannot read page: pgno=%d parent=%d err=%s", pgno, parent, err))
}
// Execute callback.
typ := readFlags(page)
if err := fn(pgno, parent, typ); err != nil {
return err
}
switch typ {
switch typ := readFlags(page); typ {
case PageTypeBranch:
if err := fn(pgno, parent, typ, nil); err != nil {
return err
}
for i, n := 0, readCellN(page); i < n; i++ {
cell := readBranchCell(page, i)
if err := tx.walkTree(cell.ChildPgno, pgno, fn); err != nil {
@ -927,18 +945,24 @@ func (tx *Tx) walkTree(pgno, parent uint32, fn func(pgno, parent, typ uint32) er
}
}
return nil
case PageTypeLeaf:
if err := fn(pgno, parent, typ, nil); err != nil {
return err
}
// Execute callback only for bitmap pages pointed to by this leaf.
for i, n := 0, readCellN(page); i < n; i++ {
if cell := readLeafCell(page, i); cell.Type == ContainerTypeBitmapPtr {
if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap); err != nil {
if err := fn(toPgno(cell.Data), pgno, PageTypeBitmap, nil); err != nil {
return err
}
}
}
return nil
default:
return fmt.Errorf("rbf.Tx.forEachTreePage(): invalid page type: pgno=%d type=%d", pgno, typ)
return fn(pgno, parent, typ, fmt.Errorf("invalid page type: pgno=%d parent=%d type=%d", pgno, parent, typ))
}
}
@ -1903,6 +1927,7 @@ func (tx *Tx) Pages(pgnos []uint32) ([]Page, error) {
// PageInfos returns meta data about all pages in the database.
func (tx *Tx) PageInfos() ([]PageInfo, error) {
var errorList ErrorList
infos := make([]PageInfo, tx.PageN())
// Read meta page info.
@ -1916,7 +1941,8 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) {
for pgno := metaInfo.RootRecordPageNo; pgno != 0; {
info, err := tx.rootRecordPageInfo(pgno)
if err != nil {
return nil, err
errorList.Append(err)
break
}
infos[pgno] = info
pgno = info.Next
@ -1924,33 +1950,34 @@ func (tx *Tx) PageInfos() ([]PageInfo, error) {
// Traverse freelist and mark pages as in-use.
if err := tx.walkPageInfo(infos, metaInfo.FreelistPageNo, "freelist"); err != nil {
return nil, err
errorList.Append(err)
}
// Traverse every b-tree and mark pages as in-use.
records, err := tx.RootRecords()
if err != nil {
return nil, err
}
errorList.Append(err)
} else {
for itr := records.Iterator(); !itr.Done(); {
name, pgno := itr.Next()
for itr := records.Iterator(); !itr.Done(); {
name, pgno := itr.Next()
if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil {
return nil, err
if err := tx.walkPageInfo(infos, pgno.(uint32), name.(string)); err != nil {
errorList.Append(err)
}
}
}
// Build page info objects for each free page.
freePageSet, err := tx.freePageSet()
if err != nil {
return nil, err
}
for pgno := range freePageSet {
infos[pgno] = &FreePageInfo{Pgno: pgno}
errorList.Append(err)
} else {
for pgno := range freePageSet {
infos[pgno] = &FreePageInfo{Pgno: pgno}
}
}
return infos, nil
return infos, errorList.Err()
}
// metaPageInfo returns page metadata for the meta page.
@ -1984,10 +2011,18 @@ func (tx *Tx) rootRecordPageInfo(pgno uint32) (*RootRecordPageInfo, error) {
}
func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error {
return tx.walkTree(root, 0, func(pgno, parent, typ uint32) error {
var errorList ErrorList
if err := tx.walkTree(root, 0, func(pgno, parent, typ uint32, err error) error {
if err != nil {
errorList.Append(err)
return nil
}
buf, _, err := tx.readPage(pgno)
if err != nil {
return err
errorList.Append(fmt.Errorf("cannot read page: pgno=%d parent=%d typ=%d err=%d", pgno, parent, typ, err))
return nil
}
switch typ {
@ -2013,12 +2048,14 @@ func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error {
Parent: parent,
Tree: name,
}
default:
vprint.PanicOn(fmt.Sprintf("unexpected page type %d for page %d", typ, pgno))
}
return nil
})
}); err != nil {
errorList.Append(err)
}
return errorList.Err()
}
// PageData returns the raw page data for a single page.
@ -2042,6 +2079,20 @@ func (tx *Tx) GetSortedFieldViewList() (fvs []txkey.FieldView, _ error) {
return
}
func (tx *Tx) DebugInfo() *TxDebugInfo {
return &TxDebugInfo{
Ptr: fmt.Sprintf("%p", tx),
Writable: tx.writable,
Stack: string(tx.stack),
}
}
type TxDebugInfo struct {
Ptr string `json:"ptr"`
Writable bool `json:"writable"`
Stack string `json:"stack,omitempty"`
}
// SnapshotReader returns a reader that provides a snapshot for the current database state.
func (tx *Tx) SnapshotReader() (io.Reader, error) {
if tx.db == nil {

31
rbf/tx_test.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"math/rand"
"os"
"path/filepath"
"strings"
"sync"
"testing"
@ -870,6 +871,36 @@ func TestTx_Check(t *testing.T) {
t.Fatalf("unexpected error: %#v", err)
}
})
t.Run("ErrBadFreelist", func(t *testing.T) {
t.Parallel()
db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-freelist"))
defer db.Close()
tx := MustBegin(t, db, false)
defer tx.Rollback()
if err, ok := tx.Check().(rbf.ErrorList); !ok {
t.Fatal("expected error list")
} else if s := err.FullError(); !strings.Contains(s, `branch cell index out of range: pgno=2 i=0 n=0`) {
t.Fatalf("unexpected error:\n%s", s)
}
})
t.Run("ErrBadBitmap", func(t *testing.T) {
t.Parallel()
db := MustOpenDBAt(t, filepath.Join("testdata", "check", "bad-bitmap"))
defer db.Close()
tx := MustBegin(t, db, false)
defer tx.Rollback()
if err, ok := tx.Check().(rbf.ErrorList); !ok {
t.Fatal("expected error list")
} else if s := err.FullError(); !strings.Contains(s, `cannot read page: pgno=65537 parent=3 err=rbf: page read out of bounds: pgno=65537 max=3`) {
t.Fatalf("unexpected error:\n%s", s)
}
})
}
func mustReadPage(tb testing.TB, path string, pgno uint32) []byte {

4
server.go
View file

@ -514,6 +514,10 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.holder.schemator = s.schemator
s.holder.sharder = s.sharder
s.holder.serializer = s.serializer
// Initial stats must be invoked after the executor obtains reference to the holder.
s.executor.InitStats()
return s, nil
}

155
server/handler_test.go
View file

@ -13,7 +13,6 @@ import (
gohttp "net/http"
"net/http/httptest"
"reflect"
"sort"
"strings"
"sync"
"testing"
@ -24,10 +23,8 @@ import (
"github.com/molecula/featurebase/v2/encoding/proto"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/pql"
pb "github.com/molecula/featurebase/v2/proto"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/test"
"google.golang.org/grpc"
)
func TestHandler_PostSchemaCluster(t *testing.T) {
@ -1472,93 +1469,93 @@ func TestClusterTranslator(t *testing.T) {
}
}
func TestQueryHistory(t *testing.T) {
cluster := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("1"),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("0"),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("2"),
)},
)
defer cluster.Close()
// func TestQueryHistory(t *testing.T) {
// cluster := test.MustRunCluster(t, 3,
// []server.CommandOption{
// server.OptCommandServerOptions(
// pilosa.OptServerNodeID("1"),
// )},
// []server.CommandOption{
// server.OptCommandServerOptions(
// pilosa.OptServerNodeID("0"),
// )},
// []server.CommandOption{
// server.OptCommandServerOptions(
// pilosa.OptServerNodeID("2"),
// )},
// )
// defer cluster.Close()
cmd := cluster.GetNode(0)
h := cmd.Handler.(*http.Handler).Handler
// cmd := cluster.GetNode(0)
// h := cmd.Handler.(*http.Handler).Handler
w := httptest.NewRecorder()
// w := httptest.NewRecorder()
test.Do(t, "POST", cmd.URL()+"/index/i0", "")
test.Do(t, "POST", cmd.URL()+"/index/i0/field/f0", "")
// test.Do(t, "POST", cmd.URL()+"/index/i0", "")
// test.Do(t, "POST", cmd.URL()+"/index/i0/field/f0", "")
gh := server.NewGRPCHandler(cmd.API)
stream := &MockServerTransportStream{}
ctx := grpc.NewContextWithServerTransportStream(context.Background(), stream)
_, err := gh.QuerySQLUnary(ctx, &pb.QuerySQLRequest{
Sql: `select * from i0`,
})
// gh := server.NewGRPCHandler(cmd.API)
// stream := &MockServerTransportStream{}
// ctx := grpc.NewContextWithServerTransportStream(context.Background(), stream)
// _, err := gh.QuerySQLUnary(ctx, &pb.QuerySQLRequest{
// Sql: `select * from i0`,
// })
if err != nil {
t.Fatalf("QuerySQLUnary failed: %v", err)
}
// if err != nil {
// t.Fatalf("QuerySQLUnary failed: %v", err)
// }
test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(0, f0=0)")
test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(3000000, f0=0)")
test.Do(t, "POST", cmd.URL()+"/index/i0/query", "TopN(f0)")
// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(0, f0=0)")
// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(3000000, f0=0)")
// test.Do(t, "POST", cmd.URL()+"/index/i0/query", "TopN(f0)")
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/query-history", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
}
// h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/query-history", nil))
// if w.Code != gohttp.StatusOK {
// t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
// }
ret := make([]pilosa.PastQueryStatus, 4)
b, err := ioutil.ReadAll(w.Body)
if err != nil {
t.Fatalf("reading: %v", err)
}
err = json.Unmarshal(b, &ret)
if err != nil {
t.Fatalf("unmarshalling: %v", err)
}
// ret := make([]pilosa.PastQueryStatus, 4)
// b, err := ioutil.ReadAll(w.Body)
// if err != nil {
// t.Fatalf("reading: %v", err)
// }
// err = json.Unmarshal(b, &ret)
// if err != nil {
// t.Fatalf("unmarshalling: %v", err)
// }
// verify result length
if len(ret) != 4 {
// each set query executes on both nodes once
// topn query gets added to history on node0 once, node1 twice
t.Fatalf("expected list of length 4, got %d\n%+v", len(ret), ret)
}
// // verify result length
// if len(ret) != 4 {
// // each set query executes on both nodes once
// // topn query gets added to history on node0 once, node1 twice
// t.Fatalf("expected list of length 4, got %d\n%+v", len(ret), ret)
// }
// verify sort order
if !sort.SliceIsSorted(ret, func(i, j int) bool {
// must match the sort in api.PastQueries
return ret[i].Start.After(ret[j].Start)
}) {
t.Fatalf("response list not sorted correctly")
}
// // verify sort order
// if !sort.SliceIsSorted(ret, func(i, j int) bool {
// // must match the sort in api.PastQueries
// return ret[i].Start.After(ret[j].Start)
// }) {
// t.Fatalf("response list not sorted correctly")
// }
// verify some response values
if ret[0].Index != "i0" {
t.Fatalf("response value for 'Index' was '%s', expected 'i0'", ret[0].Index)
}
if ret[0].Node != cluster.GetNode(0).Server.NodeID() {
t.Fatalf("response value for 'Node' was '%s', expected '%s'", ret[0].Node, cluster.GetNode(0).Server.NodeID())
}
if ret[3].PQL != "Extract(All(),Rows(f0))" {
t.Fatalf("response value for 'PQL' was '%s', expected 'Extract(All(),Rows(f0))'", ret[0].PQL)
}
if ret[3].SQL != "select * from i0" {
t.Fatalf("response value for 'SQL' was '%s', expected 'select * from i0'", ret[0].SQL)
}
if ret[0].PQL != "TopN(f0)" {
t.Fatalf("response value for 'PQL' was '%s', expected 'TopN(f0)'", ret[0].PQL)
}
}
// // verify some response values
// if ret[0].Index != "i0" {
// t.Fatalf("response value for 'Index' was '%s', expected 'i0'", ret[0].Index)
// }
// if ret[0].Node != cluster.GetNode(0).Server.NodeID() {
// t.Fatalf("response value for 'Node' was '%s', expected '%s'", ret[0].Node, cluster.GetNode(0).Server.NodeID())
// }
// if ret[3].PQL != "Extract(All(),Rows(f0))" {
// t.Fatalf("response value for 'PQL' was '%s', expected 'Extract(All(),Rows(f0))'", ret[0].PQL)
// }
// if ret[3].SQL != "select * from i0" {
// t.Fatalf("response value for 'SQL' was '%s', expected 'select * from i0'", ret[0].SQL)
// }
// if ret[0].PQL != "TopN(f0)" {
// t.Fatalf("response value for 'PQL' was '%s', expected 'TopN(f0)'", ret[0].PQL)
// }
// }
func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) {
dec := json.NewDecoder(r)

9
server/server.go
View file

@ -552,6 +552,15 @@ func (m *Command) SetupServer() error {
m.querylogger.Infof("Group with admin level access: %v", p.Admin)
m.querylogger.Infof("Permissions: %+v", p.Permissions)
// disable postgres binding if auth is enabled
m.Config.Postgres.Bind = ""
// TLS must be enabled if auth is
if m.Config.TLS.CertificatePath == "" || m.Config.TLS.CertificateKeyPath == "" || m.Config.TLS.CACertPath == "" {
return fmt.Errorf("transport layer security (TLS) is not configured properly. TLS is required when AuthN/Z is enabled, current configuration: %v", m.Config.TLS)
}
}
m.Handler, err = http.NewHandler(

86
server/server_test.go
View file

@ -17,7 +17,7 @@ import (
"testing"
"time"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/pql"
@ -26,6 +26,7 @@ import (
"github.com/molecula/featurebase/v2/test"
"github.com/molecula/featurebase/v2/testhook"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
@ -504,6 +505,30 @@ func TestClusteringNodesReplica1(t *testing.T) {
t.Fatalf("starting cluster: %v", err)
}
indexName := "idx"
fieldName := "fld"
// Create the schema.
if _, err := cluster.GetPrimary().API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil {
t.Fatalf("creating index: %v", err)
}
if _, err := cluster.GetPrimary().API.CreateField(context.Background(), indexName, fieldName); err != nil {
t.Fatalf("creating field: %v", err)
}
// Set some columns across shards to ensure that the Row query will require
// data from all nodes.
data := []string{}
for rowID := 1; rowID < 2; rowID++ {
for columnID := 1; columnID < 10; columnID++ {
data = append(data, fmt.Sprintf(`Set(%d, %s=%d)`, columnID*pilosa.ShardWidth, fieldName, rowID))
}
}
if _, err := cluster.GetPrimary().Query(t, indexName, "", strings.Join(data, "")); err != nil {
t.Fatalf("setting columns: %v", err)
}
// Shut down a node.
if err := cluster.GetNonPrimary().Command.Close(); err != nil {
t.Fatalf("closing third node: %v", err)
}
@ -513,7 +538,12 @@ func TestClusteringNodesReplica1(t *testing.T) {
}
// confirm that cluster stops accepting queries after one node closes
if _, err := cluster.GetPrimary().API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state DOWN") {
qry := &pilosa.QueryRequest{
Index: "idx",
Query: fmt.Sprintf("Row(%s=1)", fieldName),
}
if _, err := cluster.GetPrimary().API.Query(context.Background(), qry); !strings.Contains(err.Error(), "shard unavailable") {
t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
}
}
@ -540,8 +570,34 @@ func TestClusteringNodesReplica2(t *testing.T) {
}
defer cluster.Close()
indexName := "idx"
fieldName := "fld"
coord, others := cluster.GetPrimary(), cluster.GetNonPrimaries()
// Create the schema.
if _, err := coord.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{}); err != nil {
t.Fatalf("creating index: %v", err)
}
if _, err := coord.API.CreateField(context.Background(), indexName, fieldName); err != nil {
t.Fatalf("creating field: %v", err)
}
// Set some columns across shards to ensure that the Row query will require
// data from all nodes.
data := []string{}
cols := []uint64{}
for rowID := 1; rowID < 2; rowID++ {
for columnID := 1; columnID < 30; columnID++ {
col := uint64(columnID * pilosa.ShardWidth)
cols = append(cols, col)
data = append(data, fmt.Sprintf(`Set(%d, %s=%d)`, col, fieldName, rowID))
}
}
if _, err := coord.Query(t, indexName, "", strings.Join(data, "")); err != nil {
t.Fatalf("setting columns: %v", err)
}
if err := others[0].Close(); err != nil {
t.Fatalf("closing third node: %v", err)
}
@ -569,8 +625,30 @@ func TestClusteringNodesReplica2(t *testing.T) {
t.Fatalf("after closing second server: %v", err)
}
if _, err := coord.API.Query(context.Background(), &pilosa.QueryRequest{}); !strings.Contains(err.Error(), "not allowed in state DOWN") {
t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
qry := &pilosa.QueryRequest{
Index: "idx",
Query: fmt.Sprintf("Row(%s=1)", fieldName),
}
// Because we no longer block queries when the cluster is in state DOWN,
// there are cases where a DOWN cluster can still respond to a query. In
// that case, we want the test to pass. But if the unavailable node(s) cause
// the query to result in an error, we check that it's the error we expect.
resp, err := coord.API.Query(context.Background(), qry)
if err != nil {
if !strings.Contains(err.Error(), "shard unavailable") {
t.Fatalf("got unexpected error querying an incomplete cluster: %v", err)
}
} else {
if len(resp.Results) == 0 {
t.Fatal("got no results")
}
row, ok := resp.Results[0].(*pilosa.Row)
if !ok {
t.Fatalf("expected a *pilosa.Row, but got %T", resp.Results[0])
}
require.Equal(t, row.Columns(), cols)
}
}