scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into tlt/tx-comments

Browse source
This commit is contained in:
Travis Turner 2022-03-04 15:05:39 -06:00 • committed by GitHub
commit c4bc78bb36
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
703 changed files with 27348 additions and 11969 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
.artifactory/pipelines.yml
View file

@ -1,38 +0,0 @@
resources:
- name: featurebaseRepo
type: GitRepo
configuration:
# SCM integration where the repository is located
gitProvider: github_molecula_featurebase
# Repository path, including org name/repo name
path: molecula/featurebase
branches:
# Specifies which branches will trigger dependent steps
include: cicd
- name: featurebaseBuildInfo
type: BuildInfo
configuration:
sourceArtifactory: Molecula_Artifactory
buildName: featurebase_build
buildNumber: 4
pipelines:
- name: ScanGoCode
steps:
- name: scan
type: XrayScan
configuration:
failOnScan: false
inputResources:
- name: featurebaseBuildInfo
trigger: true
execution:
onStart:
- echo "Preparing for work..."
- echo "Prepping build environment"
onSuccess:
- echo "Job well done!"
onFailure:
- echo "uh oh, something went wrong"
onComplete:
- echo "Cleaning up some stuff"

602
.circleci/config.yml
View file

@ -1,305 +1,305 @@
version: 2.1
# version: 2.1
executors:
golang:
parameters:
version:
type: string
default: "1.15.8"
resource_class:
type: string
default: medium
docker:
- image: circleci/golang:<< parameters.version >>
resource_class: << parameters.resource_class >>
working_directory: /go/src/github.com/molecula/featurebase
# executors:
# golang:
# parameters:
# version:
# type: string
# default: "1.15.8"
# resource_class:
# type: string
# default: medium
# docker:
# - image: circleci/golang:<< parameters.version >>
# resource_class: << parameters.resource_class >>
# working_directory: /go/src/github.com/molecula/featurebase
commands:
add-github-auth:
steps:
- run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
- run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:"
restore-mod-cache:
steps:
- restore_cache:
key: mod-cache-{{ checksum "go.sum" }}
save-mod-cache:
steps:
- save_cache:
key: mod-cache-{{ checksum "go.sum" }}
paths:
- /go/pkg/mod/
checkout-plus:
steps:
- add-github-auth
- checkout
- restore-mod-cache
skip-if-root-unchanged:
description: "skips the parent job if the PR includes no changes to featurebase"
steps:
- run: |
ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true
echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES"
if [ -z "$ROOT_CHANGED_FILES" ] ; then
echo "halting step"
circleci step halt
fi
skip-if-lattice-unchanged:
description: "skips the parent job if the PR includes no changes to lattice"
steps:
- run: |
LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true
echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES"
if [ -z "$LATTICE_CHANGED_FILES" ] ; then
echo "halting step"
circleci step halt
fi
# commands:
# add-github-auth:
# steps:
# - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
# - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:"
# restore-mod-cache:
# steps:
# - restore_cache:
# key: mod-cache-{{ checksum "go.sum" }}
# save-mod-cache:
# steps:
# - save_cache:
# key: mod-cache-{{ checksum "go.sum" }}
# paths:
# - /go/pkg/mod/
# checkout-plus:
# steps:
# - add-github-auth
# - checkout
# - restore-mod-cache
# skip-if-root-unchanged:
# description: "skips the parent job if the PR includes no changes to featurebase"
# steps:
# - run: |
# ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true
# echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES"
# if [ -z "$ROOT_CHANGED_FILES" ] ; then
# echo "halting step"
# circleci step halt
# fi
# skip-if-lattice-unchanged:
# description: "skips the parent job if the PR includes no changes to lattice"
# steps:
# - run: |
# LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true
# echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES"
# if [ -z "$LATTICE_CHANGED_FILES" ] ; then
# echo "halting step"
# circleci step halt
# fi
jobs:
setup:
executor:
name: golang
steps:
- checkout-plus
- run: go mod download
- save-mod-cache
linter:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0
- run: make golangci-lint
go-mod-tidy:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- run: go mod tidy
- run: git diff --exit-code -- go.mod go.sum
check-changelog-label:
executor:
name: golang
steps:
- run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request
- run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e
test-build-arm:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- run: make build GOOS=linux GOARCH=arm GOARM=5
- run: make build GOOS=linux GOARCH=arm GOARM=6
- run: make build GOOS=linux GOARCH=arm GOARM=7
- run: make build GOOS=linux GOARCH=arm64
test:
parameters:
resource_class:
type: string
default: medium
golang_version:
type: string
default: "1.15.8"
shard_width:
type: string
default: "20"
test_make_target:
type: string
default: "test"
test_flags:
type: string
default: ""
goarch:
type: string
default: amd64
executor:
name: golang
version: << parameters.golang_version >>
resource_class: << parameters.resource_class >>
environment:
TMPDIR: /mnt/ramdisk
steps:
- checkout-plus
- skip-if-root-unchanged
- run: sudo apt-get update --allow-releaseinfo-change -y
- run: sudo apt-get install lsof
- run:
command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
no_output_timeout: 30m
test-external-lookup:
docker:
- image: circleci/golang:1.15.8
- image: circleci/postgres:13.2-ram
environment:
POSTGRES_PASSWORD=password
steps:
- checkout-plus
- skip-if-root-unchanged
- run: sudo apt-get update --allow-releaseinfo-change -y
- run: sudo apt-get install postgresql-client
- run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1)
- run:
command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
no_output_timeout: 30m
cluster-tests:
executor:
name: golang
steps:
- checkout-plus
- skip-if-root-unchanged
- setup_remote_docker
- run: make clustertests-build
release:
executor:
name: golang
steps:
- checkout-plus
- attach_workspace:
at: .
- setup_remote_docker:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker-release
- store_artifacts:
path: build
- persist_to_workspace:
root: .
paths: build
publish_release:
executor:
name: golang
steps:
- attach_workspace:
at: .
- run: go get github.com/tcnksm/ghr
- run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/
docker-build:
executor:
name: golang
steps:
- checkout-plus
- setup_remote_docker:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker GO_VERSION=1.15.8
- run: docker run featurebase:$(git describe --tags) help
dockerhub-upload-unstable:
executor:
name: golang
steps:
- checkout-plus
- setup_remote_docker:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker
- run: docker run featurebase:$(git describe --tags) help
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >>
dockerhub-upload-stable:
executor:
name: golang
steps:
- checkout-plus
- setup_remote_docker:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker
- run: docker run featurebase:$(git describe --tags) help
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >>
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest
# jobs:
# setup:
# executor:
# name: golang
# steps:
# - checkout-plus
# - run: go mod download
# - save-mod-cache
# linter:
# executor:
# name: golang
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0
# - run: make golangci-lint
# go-mod-tidy:
# executor:
# name: golang
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: go mod tidy
# - run: git diff --exit-code -- go.mod go.sum
# check-changelog-label:
# executor:
# name: golang
# steps:
# - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request
# - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e
# test-build-arm:
# executor:
# name: golang
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: make build GOOS=linux GOARCH=arm GOARM=5
# - run: make build GOOS=linux GOARCH=arm GOARM=6
# - run: make build GOOS=linux GOARCH=arm GOARM=7
# - run: make build GOOS=linux GOARCH=arm64
# test:
# parameters:
# resource_class:
# type: string
# default: medium
# golang_version:
# type: string
# default: "1.15.8"
# shard_width:
# type: string
# default: "20"
# test_make_target:
# type: string
# default: "test"
# test_flags:
# type: string
# default: ""
# goarch:
# type: string
# default: amd64
# executor:
# name: golang
# version: << parameters.golang_version >>
# resource_class: << parameters.resource_class >>
# environment:
# TMPDIR: /mnt/ramdisk
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: sudo apt-get update --allow-releaseinfo-change -y
# - run: sudo apt-get install lsof
# - run:
# command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >>
# no_output_timeout: 30m
# test-external-lookup:
# docker:
# - image: circleci/golang:1.15.8
# - image: circleci/postgres:13.2-ram
# environment:
# POSTGRES_PASSWORD=password
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - run: sudo apt-get update --allow-releaseinfo-change -y
# - run: sudo apt-get install postgresql-client
# - run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1)
# - run:
# command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
# no_output_timeout: 30m
# cluster-tests:
# executor:
# name: golang
# steps:
# - checkout-plus
# - skip-if-root-unchanged
# - setup_remote_docker
# - run: make clustertests
# release:
# executor:
# name: golang
# steps:
# - checkout-plus
# - attach_workspace:
# at: .
# - setup_remote_docker:
# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
# - run: make docker-release
# - store_artifacts:
# path: build
# - persist_to_workspace:
# root: .
# paths: build
# publish_release:
# executor:
# name: golang
# steps:
# - attach_workspace:
# at: .
# - run: go get github.com/tcnksm/ghr
# - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/
# docker-build:
# executor:
# name: golang
# steps:
# - checkout-plus
# - setup_remote_docker:
# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
# - run: make docker GO_VERSION=1.15.8
# - run: docker run featurebase:$(git describe --tags) help
# dockerhub-upload-unstable:
# executor:
# name: golang
# steps:
# - checkout-plus
# - setup_remote_docker:
# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
# - run: make docker
# - run: docker run featurebase:$(git describe --tags) help
# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >>
# dockerhub-upload-stable:
# executor:
# name: golang
# steps:
# - checkout-plus
# - setup_remote_docker:
# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
# - run: make docker
# - run: docker run featurebase:$(git describe --tags) help
# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >>
# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest
workflows:
build:
jobs:
- setup:
context: molecula
filters:
tags:
only: /^v.*/
- linter:
context: molecula
requires:
- setup
- go-mod-tidy:
context: molecula
requires:
- setup
- check-changelog-label:
context: molecula
requires:
- setup
- test-build-arm:
context: molecula
requires:
- setup
- test:
name: test-golang-<< matrix.golang_version >>
resource_class: large
context: molecula
requires:
- setup
matrix:
parameters:
golang_version: ["1.15.8", "1.16.10"]
- test:
name: << matrix.test_make_target >>
resource_class: xlarge
context: molecula
requires:
- setup
matrix:
parameters:
test_make_target: ["test-race"]
- test:
name: test-shardwidth-22
context: molecula
shard_width: "22"
resource_class: large
requires:
- setup
- test-external-lookup:
context: molecula
requires:
- setup
- cluster-tests:
context: molecula
requires:
- setup
- docker-build:
context: molecula
requires:
- setup
- release:
context: molecula
requires:
- setup
filters:
tags:
only: /^v.*/
- publish_release:
context: molecula
requires:
- release
filters:
tags:
only: /^v.*/
branches:
ignore: /.*/
- dockerhub-upload-unstable:
context: molecula
requires:
- setup
filters:
branches:
only: master
- dockerhub-upload-stable:
context: molecula
requires:
- setup
filters:
tags:
only: /^v.*/
branches:
ignore: /.*/
# workflows:
# build:
# jobs:
# - setup:
# context: molecula
# filters:
# tags:
# only: /^v.*/
# - linter:
# context: molecula
# requires:
# - setup
# - go-mod-tidy:
# context: molecula
# requires:
# - setup
# - check-changelog-label:
# context: molecula
# requires:
# - setup
# - test-build-arm:
# context: molecula
# requires:
# - setup
# - test:
# name: test-golang-<< matrix.golang_version >>
# resource_class: large
# context: molecula
# requires:
# - setup
# matrix:
# parameters:
# golang_version: ["1.15.8", "1.16.10"]
# - test:
# name: << matrix.test_make_target >>
# resource_class: xlarge
# context: molecula
# requires:
# - setup
# matrix:
# parameters:
# test_make_target: ["test-race"]
# - test:
# name: test-shardwidth-22
# context: molecula
# shard_width: "22"
# resource_class: large
# requires:
# - setup
# - test-external-lookup:
# context: molecula
# requires:
# - setup
# - cluster-tests:
# context: molecula
# requires:
# - setup
# - docker-build:
# context: molecula
# requires:
# - setup
# - release:
# context: molecula
# requires:
# - setup
# filters:
# tags:
# only: /^v.*/
# - publish_release:
# context: molecula
# requires:
# - release
# filters:
# tags:
# only: /^v.*/
# branches:
# ignore: /.*/
# - dockerhub-upload-unstable:
# context: molecula
# requires:
# - setup
# filters:
# branches:
# only: master
# - dockerhub-upload-stable:
# context: molecula
# requires:
# - setup
# filters:
# tags:
# only: /^v.*/
# branches:
# ignore: /.*/

9
.gitignore vendored
View file

@ -13,3 +13,12 @@ pilosa
*.dot
.idea/
.*.swp
.terraform/
*.tfstate
launch.json
.terraform.lock.hcl
__pycache__/
report.xml
outputs.json
builds/
*.tfstate.backup

446
.gitlab/.gitlab-ci.yml
View file

@ -3,44 +3,55 @@ include:
- template: Security/License-Scanning.gitlab-ci.yml
- template: Security/Dependency-Scanning.gitlab-ci.yml
.go-cache:
variables:
GOPATH: $CI_PROJECT_DIR/.go
cache:
- key: $CI_COMMIT_REF_SLUG
paths:
- .go/pkg/mod/
variables:
GOVERSION: "1.16.9"
GOVERSION: "1.16.13"
stages:
- lint
- test
- build
- integration
- gauntlet
- performance
- post build
- nonblocking
#before_script:
#- echo "before_script"
#- git version
#- go env -w GOPRIVATE=github.com/molecula
#- mkdir -p .go
#- go version
#- go env -w GO111MODULE=on
smoke build:
image: golang:$GOVERSION
stage: lint
allow_failure: false
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- echo "Let's just see if it compiles... (sometimes the linter gives unclear errors if it doesn't)"
- go build ./...
golangci-lint:
image: golangci/golangci-lint:v1.39.0
stage: lint
extends: .go-cache
allow_failure: false
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- echo "Checking for issues in new code"
- golangci-lint run -v
- golangci-lint run
go mod tidy:
stage: lint
image: golang:$GOVERSION
rules:
- if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
script:
- go mod tidy
- git diff --exit-code -- go.mod go.sum
build lattice:
stage: test
image: node:14
variables:
CI: "false"
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- cd lattice
- yarn install
@ -59,6 +70,8 @@ run jest tests:
image: node:14
variables:
CI: "true"
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- echo "Testing lattice..."
- cd lattice
@ -70,156 +83,457 @@ run jest tests:
run go tests:
stage: test
image: golang:1.16.10
extends: .go-cache
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
retry: 1
script:
- echo "Running featurebase unit tests..."
- PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
- go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./...
artifacts:
paths:
- coverage.out
- go test -timeout=30m ./...
tags:
- aws
run go tests race:
stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests.
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
retry: 1
needs: [] # don't wait to start running this.
script:
- echo "Running featurebase race tests..."
- go test -race -v -timeout=90m ./...
tags:
- aws
run go tests shardwidth22:
stage: test
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- echo "Running featurebase shardwidth22 tests..."
- go test -timeout=30m -tags=shardwidth22 ./...
tags:
- aws
# we do coverage reporting from the future tests because the json
# output is very difficult to human-read. The alternative would be to
# run the regular tests twice and also run the future tests.
run go tests future:
stage: test
image: golang:1.17.3
extends: .go-cache
image: golang:1.17.6
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
retry: 1
script:
- echo "Running featurebase unit tests..."
- PKG_LIST=$(go list ./... | grep -v internal/clustertests | paste -s -d, -)
- go test -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./...
- go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out
artifacts:
paths:
- coverage.out
run go tests with output:
stage: test
image: golang:1.16.10
script:
- echo "Running featurebase unit tests to capture JSON output..."
- go test -json > test-report.out
artifacts:
paths:
- test-report.out
tags:
- aws
upload to sonarcloud:
stage: test
stage: integration
image: sonarsource/sonar-scanner-cli:4.6
variables:
SONAR_TOKEN: $SONAR_TOKEN
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out -Dsonar.go.tests.reportPaths=test-report.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
- sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage.out,results/coverage*.out -Dsonar.go.tests.reportPaths=test-report.out,results/report* -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
needs:
- job: run go tests
- job: run go tests with output
- job: run go tests future
- job: run jest tests
- job: clustertests
build for linux amd64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- rm -r lattice
- tar -xvf lattice.tar.gz
- go get -v -u github.com/rakyll/statik
- /go/bin/statik -src=lattice
- GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
- GOOS="linux" GOARCH="amd64" go build -o roaring-migrate_linux_amd64 ./cmd/roaring-migrate
artifacts:
paths:
- featurebase_linux_amd64
- roaring-migrate_linux_amd64
build for linux arm64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- rm -r lattice
- tar -xvf lattice.tar.gz
- go get -v -u github.com/rakyll/statik
- /go/bin/statik -src=lattice
- GOOS="linux" GOARCH="arm64" make build FLAGS="-o featurebase_linux_arm64"
- GOOS="linux" GOARCH="arm64" go build -o roaring-migrate_linux_arm64 ./cmd/roaring-migrate
artifacts:
paths:
- featurebase_linux_arm64
- roaring-migrate_linux_arm64
build for darwin amd64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- rm -r lattice
- tar -xvf lattice.tar.gz
- go get -v -u github.com/rakyll/statik
- /go/bin/statik -src=lattice
- GOOS="darwin" GOARCH="amd64" make build FLAGS="-o featurebase_darwin_amd64"
- GOOS="darwin" GOARCH="amd64" go build -o roaring-migrate_darwin_amd64 ./cmd/roaring-migrate
artifacts:
paths:
- featurebase_darwin_amd64
- roaring-migrate_darwin_amd64
build for darwin arm64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- rm -r lattice
- tar -xvf lattice.tar.gz
- go get -v -u github.com/rakyll/statik
- /go/bin/statik -src=lattice
- GOOS="darwin" GOARCH="arm64" make build FLAGS="-o featurebase_darwin_arm64"
- GOOS="darwin" GOARCH="arm64" go build -o roaring-migrate_darwin_arm64 ./cmd/roaring-migrate
artifacts:
paths:
- featurebase_darwin_arm64
- roaring-migrate_darwin_arm64
package for linux amd64:
stage: build
image: golang:1.16.10
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
variables:
GOOS: "linux"
GOARCH: "amd64"
script:
- echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
- apt update && apt install nfpm
- apt update && apt install nfpm=2.11.3
- make package
artifacts:
paths:
- "*.deb"
- "*.rpm"
# Build a FB Docker image with CI/CD and push to the GitLab registry.
build container fb:
image: docker:stable
package for linux arm64:
stage: build
image: golang:$GOVERSION
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
variables:
GOOS: "linux"
GOARCH: "arm64"
script:
- echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | tee /etc/apt/sources.list.d/goreleaser.list
- apt update && apt install nfpm=2.11.3
- make package
artifacts:
paths:
- "*.deb"
- "*.rpm"
build amd container fb:
stage: build
needs:
- "build for linux amd64"
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
before_script:
- echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
script:
- tag=${CI_REGISTRY_IMAGE}/server:${CI_COMMIT_REF_SLUG}
- docker build --build-arg GO_VERSION=$GOVERSION -t $tag -f .gitlab/Dockerfile .
- tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
- docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 -t $tag -f .gitlab/Dockerfile .
- docker push $tag
- echo Created docker featurebase image with tag "$tag"
# deploy EC2 instance, configure and run featurebase
deploy node for linux amd64:
build arm container fb:
stage: build
needs:
- "build for linux arm64"
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
before_script:
- echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
script:
- tag=${CI_REGISTRY_IMAGE}/featurebase:linux-arm64-${CI_COMMIT_REF_SLUG}
- docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=arm64 -t $tag -f .gitlab/Dockerfile .
- docker push $tag
- echo Created docker featurebase image with tag "$tag"
# clustertests doesn't run in docker, and requires several things to be set up on the runner to work:
# 1. Install Go, make sure it's on the path
# 2. Make sure "make" is installed
# 3. make sure docker/docker-compose is installed
# 4. make sure the git config is done `git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"`
# 5. Add deploy key github.com/molecula/featurebase/settings/keys and add public key in .ssh folder of gitlab-runner user
# TODO: (I think) get clustertests coverage added to coverage report
clustertests:
variables:
PROJECT: clustertests_${CI_CONCURRENT_ID}
stage: integration
tags:
- shell
retry: 1
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- rm -rf internal/clustertests/results && mkdir -p internal/clustertests/results && chown gitlab-runner:gitlab-runner internal/clustertests/results
- make clustertests
- mv internal/clustertests/results/ results/
artifacts:
paths:
- results/coverage*.out
authclustertests:
variables:
PROJECT: authclustertests_${CI_CONCURRENT_ID}
stage: integration
retry: 1
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
script:
- rm -rf internal/clustertests/results && mkdir -p internal/clustertests/results && chown gitlab-runner:gitlab-runner internal/clustertests/results
- make authclustertests
- rm -rf internal/clustertests/results
external lookup tests:
stage: integration
image: golang:$GOVERSION
# TODO: no rules here, do we need to add the rules line?
variables:
POSTGRES_DB: $POSTGRES_DB
POSTGRES_USER: $POSTGRES_USER
POSTGRES_PASSWORD: $POSTGRES_PASSWORD
POSTGRES_HOST_AUTH_METHOD: trust
services:
- postgres:13.5
script:
- apt-get update --allow-releaseinfo-change -y
- apt-get install -y postgresql-client
- go test . -run "^TestExternalLookup" -externalLookupDSN postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB?sslmode=disable
smoke test:
stage: integration
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
variables:
PROFILE: "default"
AWS_SSH_PRIVATE_KEY: $AWS_SSH_PRIVATE_KEY
PROFILE: "service-terraform"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
TF_VAR_cluster_prefix: ""
tags:
- aws
- docker
- fbsmoke
rules:
- if: '$CI_PIPELINE_SOURCE == "push"'
before_script:
- aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
- aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
- aws configure set region "us-east-2"
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- apt-get update && apt-get install terraform
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
- aws configure set region "us-east-2" --profile $PROFILE
- aws configure set aws_profile $PROFILE
- echo $AWS_SSH_PRIVATE_KEY > gitlab-featurebase-dev.pem
- chmod 400 gitlab-featurebase-dev.pem
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
- chmod 400 gitlab-featurebase-ci.pem
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
- eval `ssh-agent -s`
- mkdir -p ~/.ssh
- echo "$AWS_SSH_PRIVATE_KEY" | ssh-add -
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
- chmod 700 /root/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- apt update && apt -y install jq
- apt update && apt -y install jq wget
- wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
- tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
- export PATH=$PATH:/usr/local/go/bin
- TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
script:
- ./qa/scripts/deployNode.sh $PROFILE
- ./qa/scripts/setupSmokeTest.sh
- ./qa/scripts/testSmokeTest.sh
after_script:
- ./qa/scripts/teardownSmokeTest.sh
needs:
- job: build for linux arm64
artifacts:
when: always
paths:
- report.xml
reports:
junit: report.xml
gauntlet:
stage: gauntlet
timeout: 4h
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
variables:
FBCI_PROFILE: "service-terraform"
INFRA_PROFILE: "service-gitlab"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
ASG_NAME: "gitlab-runners"
TF_VAR_cluster_prefix: ""
tags:
- aws
- docker
- fbsmoke
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
before_script:
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- apt-get update && apt-get install terraform
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $FBCI_PROFILE
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $FBCI_PROFILE
- aws configure set region "us-east-2" --profile $FBCI_PROFILE
- aws configure set aws_profile $FBCI_PROFILE
- aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
- aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
- aws configure set region "us-east-2" --profile $INFRA_PROFILE
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
- chmod 400 gitlab-featurebase-ci.pem
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
- chmod 700 /root/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- apt update && apt -y install jq wget
- wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
- tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
- export PATH=$PATH:/usr/local/go/bin
- TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
- export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
script:
- ./qa/scripts/setupSamsungGauntlet.sh
- ./qa/scripts/testSamsungGauntlet.sh
after_script:
- ./qa/scripts/teardownSamsungGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
- export INSTANCE_ID=$(cat instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
s3 dump:
stage: post build
variables:
PROFILE: "service-fb-ci"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
tags:
- shell
rules:
- if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
script:
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
- aws configure set region "us-east-2"
- aws configure set aws_profile $PROFILE
- aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_amd64
- aws s3 cp featurebase_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_amd64
- aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_amd64
- aws s3 cp roaring-migrate_linux_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_amd64
- aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_linux_arm64
- aws s3 cp featurebase_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64
- aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_linux_arm64
- aws s3 cp roaring-migrate_linux_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_linux_arm64
- aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_amd64
- aws s3 cp featurebase_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_amd64
- aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_amd64
- aws s3 cp roaring-migrate_darwin_amd64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_amd64
- aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
- aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
- aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64
- aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_arm64
needs:
- job: build for darwin amd64
- job: build for darwin arm64
- job: build for linux amd64
- job: build for linux arm64
perf_able:
stage: performance
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"'
trigger:
include: .gitlab/.perf-able-gitlab-ci.yml
variables:
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
s3 dump tag:
stage: post build
variables:
PROFILE: "service-fb-ci"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
LOCATION: molecula-artifact-storage/featurebase/_tags
tags:
- shell
rules:
- if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")'
script:
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
- aws configure set region "us-east-2"
- aws configure set aws_profile $PROFILE
- |
for goos in "darwin" "linux"; do
for goarch in "amd64" "arm64"; do
dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch}
echo "Directory ${dir}"
mkdir $dir
mv featurebase_${goos}_${goarch} ${dir}/featurebase
mv roaring-migrate_${goos}_${goarch} ${dir}/roaring-migrate
cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/
tar cvzf ${dir}.tar.gz ${dir}
aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
done
done
needs:
- job: build for darwin amd64
- job: build for darwin arm64
- job: build for linux amd64
- job: build for linux arm64

61
.gitlab/.perf-able-gitlab-ci.yml Normal file
View file

@ -0,0 +1,61 @@
stages:
- performance
perf_able:
stage: performance
timeout: 2h
image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest
variables:
PROFILE: "service-terraform"
INFRA_PROFILE: "service-gitlab"
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
ASG_NAME: "gitlab-runners"
TF_VAR_cluster_prefix: ""
tags:
- aws
- docker
- fbsmoke
before_script:
- apt-get update && apt-get install -y gnupg software-properties-common curl git
- curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- apt-get update && apt-get install terraform
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE
- aws configure set region "us-east-2" --profile $PROFILE
- aws configure set aws_profile $PROFILE
- aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE
- aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE
- aws configure set region "us-east-2" --profile $INFRA_PROFILE
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
- chmod 400 gitlab-featurebase-ci.pem
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
- eval $(ssh-agent -s)
- mkdir -p ~/.ssh
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
- chmod 400 /root/.ssh/gitlab-featurebase-ci.pem
- echo "$AWS_FBCI_SSH_KEY" | ssh-add -
- chmod 700 /root/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- apt update && apt -y install jq wget
- wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz
- tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz
- export PATH=$PATH:/usr/local/go/bin
- TF_VAR_cluster_prefix="able-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)"
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
- export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
script:
- ./qa/scripts/perf/able/ableSetup.sh
- ./qa/scripts/perf/able/ableTest.sh
after_script:
- ./qa/scripts/perf/able/ableTeardown.sh || true
- export INSTANCE_ID=$(cat instance_id)
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
needs:
- pipeline: $PARENT_PIPELINE_ID
job: build for linux arm64
- pipeline: $PARENT_PIPELINE_ID
job: build for linux amd64

8
.gitlab/Dockerfile
View file

@ -3,12 +3,14 @@ FROM alpine:3.14.2
LABEL maintainer "dev@molecula.com"
LABEL org.opencontainers.image.authors="dev@molecula.com"
WORKDIR /featurebase
ARG ARCH
WORKDIR /
RUN apk add --no-cache curl jq
COPY NOTICE .
COPY featurebase_linux_amd64 .
COPY featurebase_linux_$ARCH featurebase
RUN chmod ugo+x .
EXPOSE 10101
@ -19,4 +21,4 @@ ENV PILOSA_BIND 0.0.0.0:10101
ENV PILOSA_BIND_GRPC 0.0.0.0:20101
ENTRYPOINT ["/featurebase"]
CMD ["server"]
CMD ["server"]

14
Dockerfile-clustertests
View file

@ -7,8 +7,6 @@ LABEL maintainer "dev@pilosa.com"
COPY . /go/src/github.com/molecula/featurebase/
RUN cd /go/src/github.com/molecula/featurebase \
&& make install FLAGS="-a -mod=vendor"
# download pumba for fault injection
ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
@ -18,7 +16,15 @@ RUN chmod +x /pumba
RUN apt update
RUN apt install -y docker.io
RUN cp /go/bin/featurebase /featurebase
# add docker-compose so tests can use it for stuff
ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
RUN chmod +x /usr/local/bin/docker-compose
# generate an instrumented binary to allow for calculating code coverage for clustertests
# the entrypoint for the binary is TestRunMain, which is wrapper for main
RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
COPY NOTICE /NOTICE
@ -26,4 +32,4 @@ EXPOSE 10101
VOLUME /data
ENTRYPOINT ["bash", "-c"]
CMD ["/featurebase", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

35
Dockerfile-clustertests-client Normal file
View file

@ -0,0 +1,35 @@
# This Dockerfile is used for cluster testing - it produces a much larger image
# and includes all of Go as well as some utilities.
FROM golang:1.16
LABEL maintainer "dev@pilosa.com"
COPY . /go/src/github.com/molecula/featurebase/
# download pumba for fault injection
ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
RUN chmod +x /pumba
# add docker client to pause/unpause nodes
RUN apt update
RUN apt install -y docker.io
# add docker-compose so tests can use it for stuff
ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
RUN chmod +x /usr/local/bin/docker-compose
RUN cd /go/src/github.com/molecula/featurebase/cmd/featurebase && \
go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase && \
cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
COPY NOTICE /NOTICE
COPY ./internal/clustertests /go/src/github.com/molecula/featurebase/internal/clustertests
EXPOSE 10101
VOLUME /data
ENTRYPOINT ["bash", "-c"]
CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]

76
Makefile
View file

@ -1,7 +1,6 @@
.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf
CLONE_URL=github.com/pilosa/pilosa
MOD_VERSION=v2
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
VARIANT = Molecula
GO=go
@ -13,12 +12,14 @@ BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
SHARD_WIDTH = 20
COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
LDFLAGS="-X github.com/molecula/featurebase/v2.Version=$(VERSION) -X github.com/molecula/featurebase/v2.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v2.Variant=$(VARIANT) -X github.com/molecula/featurebase/v2.Commit=$(COMMIT) -X github.com/molecula/featurebase/v2.TrialDeadline=$(TRIAL_DEADLINE)"
LDFLAGS="-X github.com/molecula/featurebase/v3.Version=$(VERSION) -X github.com/molecula/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v3.Variant=$(VARIANT) -X github.com/molecula/featurebase/v3.Commit=$(COMMIT) -X github.com/molecula/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)"
GO_VERSION=1.16.10
DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
TEST_TAGS = roaringparanoia
UNAME := $(shell uname -s)
TEST_TIMEOUT=30m
RACE_TEST_TIMEOUT=90m
ifeq ($(UNAME), Darwin)
IS_MACOS:=1
else
@ -45,11 +46,11 @@ version:
# Run test suite
test:
$(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v
$(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(TEST_TIMEOUT)
# Run test suite with race flag
test-race:
CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v
CGO_ENABLED=1 $(GO) test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout $(RACE_TEST_TIMEOUT) -v
testv: topt testvsub
@ -64,7 +65,7 @@ testvsub:
set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \
echo; echo "___ testing subpkg $$i"; \
cd $$i; pwd; \
$(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \
$(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout $(RACE_TEST_TIMEOUT) || break; \
echo; echo "999 done testing subpkg $$i"; \
cd ..; \
done
@ -73,14 +74,11 @@ testvsub-race:
set -e; for i in boltdb client ctl http pg pql rbf roaring server sql txkey; do \
echo; echo "___ testing subpkg $$i -race"; \
cd $$i; pwd; \
CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \
CGO_ENABLED=1 $(GO) test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout $(RACE_TEST_TIMEOUT) || break; \
echo; echo "999 done testing subpkg $$i -race"; \
cd ..; \
done
tour:
./tournament.sh
bench:
$(GO) test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
@ -139,22 +137,33 @@ package:
nfpm package --packager deb --target featurebase_$(VERSION_ID).deb
nfpm package --packager rpm --target featurebase_$(VERSION_ID).rpm
# try (e.g.) internal/clustertests/docker-compose-replication2.yml
DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
# We allow setting a custom docker-compose "project". Multiple of the
# same docker-compose environment can exist simultaneously as long as
# they use different projects (the project name is prepended to
# container names and such). This is useful in a CI environment where
# we might be running multiple instances of the tests concurrently.
PROJECT ?= clustertests
DOCKER_COMPOSE = docker-compose -p $(PROJECT)
# Run cluster integration tests using docker. Requires docker daemon to be
# running. This will catch changes to internal/clustertests/*.go, but if you
# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
# pilosa image.
# running and docker-compose to be installed.
clustertests: vendor
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) build
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
# Like clustertests, but rebuilds all images.
clustertests-build: vendor
docker-compose -f $(DOCKER_COMPOSE) down -v
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
# Run the cluster tests with authentication enabled
AUTH_ARGS="-c /go/src/github.com/molecula/featurebase/internal/clustertests/testdata/featurebase.conf"
authclustertests: vendor
$(eval PROJECT=authclustertests)
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml build
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
PROJECT=$(PROJECT) ENABLE_AUTH=1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
CLUSTERTESTS_FB_ARGS=$(AUTH_ARGS) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down
# Install Pilosa
install:
@ -170,11 +179,11 @@ build-lattice:
# `go generate` protocol buffers
generate-protoc: require-protoc require-protoc-gen-gofast
$(GO) generate github.com/molecula/featurebase/v2/pb
$(GO) generate github.com/molecula/featurebase/v3/pb
# `go generate` statik assets (lattice UI)
generate-statik: build-lattice require-statik
$(GO) generate github.com/molecula/featurebase/v2/statik
$(GO) generate github.com/molecula/featurebase/v3/statik
# `go generate` statik assets (lattice UI) in Docker
generate-statik-docker: build-lattice
@ -182,7 +191,7 @@ generate-statik-docker: build-lattice
# `go generate` stringers
generate-stringer:
$(GO) generate github.com/molecula/featurebase/v2
$(GO) generate github.com/molecula/featurebase/v3
generate-pql: require-peg
cd pql && peg -inline pql.peg && cd ..
@ -193,7 +202,7 @@ generate-proto-grpc: require-protoc require-protoc-gen-go
# TODO: Modify above commands and remove the below mv if possible.
# See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt
# I couldn't get it to work during development - Cody
cp -r proto/github.com/molecula/featurebase/v2/proto/ proto/
cp -r proto/github.com/molecula/featurebase/v3/proto/ proto/
rm -rf proto/github.com
# `go generate` all needed packages
@ -250,20 +259,20 @@ pilosa-fsck:
# Run Pilosa tests inside Docker container
docker-test:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./...
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -timeout $(TEST_TIMEOUT) ./...
# Must use bash in order to -o pipefail; otherwise the tee will hide red tests.
# run top tests, not subdirs. print summary red/green after.
# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
topt:
mv log.topt.roar log.topt.roar.prev || true
$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout 60m -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
$(eval SHELL:=/bin/bash) set -o pipefail; $(GO) test -v -timeout $(RACE_TEST_TIMEOUT) -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
@echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
@echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' | wc -l
topt-race:
mv log.topt.race log.topt.race.prev || true
$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout 60m -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
$(eval SHELL:=/bin/bash) set -o pipefail; CGO_ENABLED=1 $(GO) test -race -timeout $(RACE_TEST_TIMEOUT) -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
@echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
@echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' | wc -l
@ -313,7 +322,7 @@ require-%:
install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
install-statik:
go get -u github.com/rakyll/statik
go install github.com/rakyll/statik@latest
install-stringer:
GO111MODULE=off $(GO) get -u golang.org/x/tools/cmd/stringer
@ -338,14 +347,5 @@ install-gometalinter:
GO111MODULE=off gometalinter --install
GO111MODULE=off $(GO) get github.com/remyoudompheng/go-misc/deadcode
test-txstore-rbf:
PILOSA_STORAGE_BACKEND=rbf $(MAKE) testv-race
# WARNING: This feature is no longer being tested regularly in CI. The test is
# very slow and very expensive, and we're not sure it actually provides useful
# information now.
test-txstore-rbf_bolt:
PILOSA_STORAGE_BACKEND=rbf_bolt $(MAKE) testv-race
test-external-lookup:
$(GO) test . -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -run ^TestExternalLookup$$ -externalLookupDSN $(EXTERNAL_LOOKUP_DSN)

416
api.go
View file

@ -21,15 +21,16 @@ import (
"sync"
"time"
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/rbf"
//"github.com/molecula/featurebase/v2/pg"
"github.com/molecula/featurebase/v2/pql"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/stats"
"github.com/molecula/featurebase/v2/topology"
"github.com/molecula/featurebase/v2/tracing"
//"github.com/molecula/featurebase/v3/pg"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/topology"
"github.com/molecula/featurebase/v3/tracing"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -49,9 +50,6 @@ type API struct {
importWorkerPoolSize int
importWork chan importJob
usageCache *usageCache
schemaDetailsOn bool
Serializer Serializer
}
@ -72,14 +70,6 @@ func OptAPIServer(s *Server) apiOption {
}
}
// Used to configure API option: schemaDetailsOn
func OptAPISchemaDetailsOn(isOn bool) apiOption {
return func(a *API) error {
a.schemaDetailsOn = isOn
return nil
}
}
func OptAPIImportWorkerPoolSize(size int) apiOption {
return func(a *API) error {
a.importWorkerPoolSize = size
@ -130,9 +120,16 @@ func (api *API) SetAPIOptions(opts ...apiOption) error {
var validAPIMethods = map[disco.ClusterState]map[apiMethod]struct{}{
disco.ClusterStateStarting: methodsCommon,
disco.ClusterStateNormal: appendMap(methodsCommon, methodsNormal),
disco.ClusterStateDegraded: appendMap(methodsCommon, methodsDegraded),
// Ideally, this would be just `appendMap(methodsCommon, methodsDegraded)`,
// but in an attempt to reduce the influence that state (determined by etcd)
// has on a node under load, this is set to effectively allow all requests
// in a DEGRADED state.
disco.ClusterStateDegraded: appendMap(methodsCommon, methodsNormal),
disco.ClusterStateResizing: appendMap(methodsCommon, methodsResizing),
disco.ClusterStateDown: methodsCommon,
// Ideally, this would be just `methodsCommon`, but in an attempt to reduce
// the influence that state (determined by etcd) has on a node under load,
// this is set to effectively allow all requests in a DOWN state.
disco.ClusterStateDown: appendMap(methodsCommon, methodsNormal),
}
func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} {
@ -239,7 +236,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
}
// Create index.
index, err := api.holder.CreateIndexAndBroadcast(cim)
index, err := api.holder.CreateIndexAndBroadcast(ctx, cim)
if err != nil {
return nil, errors.Wrap(err, "creating index")
}
@ -331,11 +328,17 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
}
// Create field.
field, err := index.CreateFieldAndBroadcast(cfm)
field, err := index.CreateField(fieldName, opts...)
if err != nil {
return nil, errors.Wrap(err, "creating field")
}
// Send the create field message to all nodes. We do this *outside* the
// CreateField logic so we're not blocking on it.
if err := api.holder.sendOrSpool(cfm); err != nil {
return nil, errors.Wrap(err, "sending CreateField message")
}
api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
return field, nil
}
@ -930,256 +933,6 @@ func (api *API) PrimaryNode() *topology.Node {
return snap.PrimaryFieldTranslationNode()
}
// Cache of disk usage statistics
type usageCache struct {
data map[string]NodeUsage
refreshInterval time.Duration
lastUpdated time.Time
resetTrigger chan bool
lastCalcDuration time.Duration
waitMultiplier float64
disable bool
muCalculate sync.Mutex
muAssign sync.Mutex
}
var usageCacheMinDuration = 5 * time.Second // If usage takes less than this duration to calculate, don't use the cache.
var usageCacheMinInterval = time.Hour // Refresh interval is forced to be >= this duration.
var usageCacheInitialInterval = time.Hour // Refresh interval starts with this duration.
// NodeUsage represents all usage measurements for one node.
type NodeUsage struct {
Disk DiskUsage `json:"diskUsage"`
Memory MemoryUsage `json:"memoryUsage"`
LastUpdated time.Time `json:"lastUpdated"`
}
// DiskUsage represents the storage space used on disk by one node.
type DiskUsage struct {
Capacity uint64 `json:"capacity,omitempty"`
TotalUse uint64 `json:"totalInUse"`
IndexUsage map[string]IndexUsage `json:"indexes"`
}
// IndexUsage represents the storage space used on disk by one index, on one node.
type IndexUsage struct {
Total uint64 `json:"total"`
IndexKeys uint64 `json:"indexKeys"`
FieldKeysTotal uint64 `json:"fieldKeysTotal"`
Fragments uint64 `json:"fragments"`
Metadata uint64 `json:"metadata"`
Fields map[string]FieldUsage `json:"fields"`
}
// FieldUsage represents the storage space used on disk by one field, on one node
type FieldUsage struct {
Total uint64 `json:"total"`
Fragments uint64 `json:"fragments"`
Keys uint64 `json:"keys"`
Metadata uint64 `json:"metadata"`
}
// MemoryUsage represents the memory used by one node.
type MemoryUsage struct {
Capacity uint64 `json:"capacity"`
TotalUse uint64 `json:"totalInUse"`
}
// Returns disk usage from cache if cache is large. It will recalculate on the spot if the last cacluation was under 5 seconds.
func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.Usage")
defer span.Finish()
if api.usageCache.disable {
resp := make(map[string]NodeUsage)
return resp, nil
}
api.usageCache.muAssign.Lock()
lastCalc := api.usageCache.lastCalcDuration
api.usageCache.muAssign.Unlock()
if lastCalc < usageCacheMinDuration {
err := api.ResetUsageCache()
if err != nil {
api.server.logger.Infof("could not reset usageCache: %s", err)
}
}
api.usageCache.muAssign.Lock()
lastUpdated := api.usageCache.lastUpdated
api.usageCache.muAssign.Unlock()
if lastUpdated == (time.Time{}) {
api.calculateUsage()
}
if !remote {
api.requestUsageOfNodes()
}
return api.usageCache.data, nil
}
// Makes a ui/usage request for each node in cluster to calculates its usage and adds it to the cache
func (api *API) requestUsageOfNodes() {
nodes := api.cluster.Nodes()
for _, node := range nodes {
if node.ID == api.server.nodeID {
continue
}
nodeUsage, err := api.server.defaultClient.GetNodeUsage(context.Background(), &node.URI)
if err != nil {
api.server.logger.Infof("couldn't collect disk usage from %s: %s", node.URI, err)
}
api.usageCache.muAssign.Lock()
api.usageCache.data[node.ID] = nodeUsage[node.ID]
api.usageCache.muAssign.Unlock()
}
}
// Calculates disk usage from scratch if cache has expired for each index and stores the results in the usage cache
func (api *API) calculateUsage() {
api.usageCache.muCalculate.Lock()
defer api.usageCache.muCalculate.Unlock()
api.server.wg.Add(1)
defer api.server.wg.Done()
api.usageCache.muAssign.Lock()
lastUpdated := api.usageCache.lastUpdated
api.usageCache.muAssign.Unlock()
if time.Since(lastUpdated) <= api.usageCache.refreshInterval {
return
}
indexDetails, nodeMetadataBytes, err := api.holder.Txf().IndexUsageDetails(api.isClosing)
if err != nil {
api.server.logger.Infof("couldn't get index usage details: %s", err)
}
if api.isClosing() {
return
}
totalSize := nodeMetadataBytes
for _, s := range indexDetails {
totalSize += s.Total
}
// NOTE: these errors are ignored in api.Info(), but checked here
si := api.server.systemInfo
diskCapacity, err := si.DiskCapacity(api.holder.path)
if err != nil {
api.server.logger.Infof("couldn't read disk capacity: %s", err)
}
memoryCapacity, err := si.MemTotal()
if err != nil {
api.server.logger.Infof("couldn't read memory capacity: %s", err)
}
memoryUse, err := si.MemUsed()
if err != nil {
api.server.logger.Infof("couldn't read memory usage: %s", err)
}
lastUpdated = time.Now()
// Insert into result.
nodeUsage := NodeUsage{
Disk: DiskUsage{
Capacity: diskCapacity,
TotalUse: totalSize,
IndexUsage: indexDetails,
},
Memory: MemoryUsage{
Capacity: memoryCapacity,
TotalUse: memoryUse,
},
LastUpdated: lastUpdated,
}
api.usageCache.muAssign.Lock()
api.usageCache.data = make(map[string]NodeUsage)
api.usageCache.data[api.server.nodeID] = nodeUsage
api.usageCache.lastUpdated = lastUpdated
api.usageCache.muAssign.Unlock()
}
// Periodically calculates disk/memory usage in terms of the duty cycle. The duty cycle represents the percentage of
// time that is spent recalculating this cache. It is specified relatively, rather than by a set interval, because
// scans can take an unpredictably long time.
func (api *API) RefreshUsageCache(dutyCycle float64) {
if dutyCycle == 0 {
api.server.logger.Warnf("usage-duty-cycle set to 0, usage cache and /ui/usage endpoint are disabled")
api.usageCache = &usageCache{
disable: true,
}
return
}
trigger := make(chan bool)
defer close(trigger)
multiplier := 100/dutyCycle - 1
api.usageCache = &usageCache{
data: make(map[string]NodeUsage),
refreshInterval: usageCacheInitialInterval,
resetTrigger: trigger,
lastCalcDuration: 0,
waitMultiplier: multiplier,
}
api.server.logger.Infof("monitoring resource usage with duty cycle %v%%\n", dutyCycle)
for {
start := time.Now()
api.calculateUsage()
api.setRefreshInterval(time.Since(start))
api.server.logger.Infof("updated resource usage cache at %v, took %v, next update in %v\n", api.usageCache.lastUpdated.Format(time.RFC3339), api.usageCache.lastCalcDuration.Truncate(time.Millisecond), api.usageCache.refreshInterval.Truncate(100*time.Millisecond))
select {
case <-trigger:
continue
case <-api.server.closing:
return
case <-time.After(api.usageCache.refreshInterval):
continue
}
}
}
// Refresh interval set in relation to how long the last calculation took.
func (api *API) setRefreshInterval(dur time.Duration) {
refresh := time.Duration(float64(dur) * api.usageCache.waitMultiplier)
if refresh < usageCacheMinInterval {
refresh = usageCacheMinInterval
}
api.usageCache.muAssign.Lock()
api.usageCache.refreshInterval = refresh
api.usageCache.lastCalcDuration = dur
api.usageCache.muAssign.Unlock()
}
// Resets the lastUpdated time and awakens RefreshUsageCache()
func (api *API) ResetUsageCache() error {
if api.usageCache != nil {
api.usageCache.muAssign.Lock()
api.usageCache.lastUpdated = time.Time{}
api.usageCache.muAssign.Unlock()
} else {
return errors.New("invalidating cache: cache not initialized")
}
api.usageCache.resetTrigger <- true
return nil
}
// isClosing returns true if the server is shutting down.
func (api *API) isClosing() bool {
select {
case <-api.server.closing:
return true
default:
return false
}
}
// RecalculateCaches forces all TopN caches to be updated.
// This is done internally within a TopN query, but a user may want to do it ahead of time?
func (api *API) RecalculateCaches(ctx context.Context) error {
@ -1264,38 +1017,6 @@ func (api *API) Schema(ctx context.Context, withViews bool) ([]*IndexInfo, error
return api.holder.limitedSchema()
}
// SchemaDetails returns information about each index in Pilosa including which
// fields they contain. Additional field information such as cardinality unless
// turned off via the schemaDetailsOn cli option.
func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.Schema")
defer span.Finish()
schema, err := api.holder.Schema()
if err != nil {
return nil, errors.Wrap(err, "getting schema")
}
if !api.schemaDetailsOn {
return schema, nil
}
for _, index := range schema {
for _, field := range index.Fields {
q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name)
req := QueryRequest{Index: index.Name, Query: q}
resp, err := api.query(ctx, &req)
if err != nil {
return schema, errors.Wrapf(err, "querying cardinality (%s/%s)", index.Name, field.Name)
}
if len(resp.Results) == 0 {
continue
}
if card, ok := resp.Results[0].(uint64); ok {
field.Cardinality = &card
}
}
}
return schema, nil
}
// ApplySchema takes the given schema and applies it across the
// cluster (if remote is false), or just to this node (if remote is
// true). This is designed for the use case of replicating a schema
@ -2299,10 +2020,6 @@ func (api *API) Info() serverInfo {
}
}
func (api *API) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error) {
return api.holder.Inspect(ctx, req)
}
// GetTranslateEntryReader provides an entry reader for key translation logs starting at offset.
func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "API.GetTranslateEntryReader")
@ -2506,24 +2223,24 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du
return nil, errors.Wrap(err, "validating api method")
}
t, err := api.server.StartTransaction(ctx, id, timeout, exclusive, remote)
if exclusive {
switch err {
case nil:
switch err {
case nil:
if exclusive {
api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0)
case ErrTransactionExclusive:
api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
}
if t.Active {
api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
}
} else {
switch err {
case nil:
} else {
api.holder.Stats.Count(MetricTransactionStart, 1, 1.0)
case ErrTransactionExclusive:
}
case ErrTransactionExclusive:
if exclusive {
api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
} else {
api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0)
}
}
if exclusive && t != nil && t.Active {
api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
}
return t, err
}
@ -2753,8 +2470,8 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
for _, flv := range flvs {
fld := idx.field(flv.Field)
view, ok := fld.viewMap[flv.View]
if !ok {
view := fld.view(flv.View)
if view == nil {
view, err = fld.createViewIfNotExists(flv.View)
if err != nil {
return err
@ -3156,6 +2873,21 @@ func (api *API) Plan(ctx context.Context, q string) (*Stmt, error) {
return api.server.PlanSQL(ctx, q)
}
func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo {
infos := make(map[string]*rbf.DebugInfo)
for key, dbShard := range api.holder.Txf().dbPerShard.Flatmap {
wrapper, ok := dbShard.W.(*RbfDBWrapper)
if !ok {
continue
}
skey := fmt.Sprintf("%s/%d", key.index, key.shard)
infos[skey] = wrapper.db.DebugInfo()
}
return infos
}
type serverInfo struct {
ShardWidth uint64 `json:"shardWidth"`
ReplicaN int `json:"replicaN"`
@ -3233,24 +2965,24 @@ var methodsResizing = map[apiMethod]struct{}{
apiSchema: {},
}
var methodsDegraded = map[apiMethod]struct{}{
apiExportCSV: {},
apiFragmentBlockData: {},
apiFragmentBlocks: {},
apiField: {},
apiIndex: {},
apiQuery: {},
apiRecalculateCaches: {},
apiRemoveNode: {},
apiShardNodes: {},
apiSchema: {},
apiViews: {},
apiStartTransaction: {},
apiFinishTransaction: {},
apiTransactions: {},
apiGetTransaction: {},
apiActiveQueries: {},
}
// var methodsDegraded = map[apiMethod]struct{}{
// apiExportCSV: {},
// apiFragmentBlockData: {},
// apiFragmentBlocks: {},
// apiField: {},
// apiIndex: {},
// apiQuery: {},
// apiRecalculateCaches: {},
// apiRemoveNode: {},
// apiShardNodes: {},
// apiSchema: {},
// apiViews: {},
// apiStartTransaction: {},
// apiFinishTransaction: {},
// apiTransactions: {},
// apiGetTransaction: {},
// apiActiveQueries: {},
// }
var methodsNormal = map[apiMethod]struct{}{
apiCreateField: {},

4
api/client/grpc.go
View file

@ -6,8 +6,8 @@ import (
"crypto/tls"
"sync"
"github.com/molecula/featurebase/v2/logger"
pb "github.com/molecula/featurebase/v2/proto"
"github.com/molecula/featurebase/v3/logger"
pb "github.com/molecula/featurebase/v3/proto"
"github.com/pkg/errors"
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"

355
api_test.go
View file

@ -4,23 +4,33 @@ package pilosa_test
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"math/rand"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"testing"
"time"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/boltdb"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/shardwidth"
"github.com/molecula/featurebase/v2/test"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
"github.com/golang-jwt/jwt"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/shardwidth"
"github.com/molecula/featurebase/v3/test"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
"golang.org/x/sync/errgroup"
)
func TestAPI_Import(t *testing.T) {
@ -30,21 +40,21 @@ func TestAPI_Import(t *testing.T) {
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -216,19 +226,19 @@ func TestAPI_ImportValue(t *testing.T) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -523,7 +533,7 @@ func TestAPI_Ingest(t *testing.T) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -642,7 +652,7 @@ func BenchmarkIngest(b *testing.B) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -703,7 +713,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
@ -948,29 +958,6 @@ func TestAPI_IDAlloc(t *testing.T) {
})
}
func TestAPI_SchemaDetailsOff(t *testing.T) {
cluster := test.MustRunCluster(t, 2)
defer cluster.Close()
cmd := cluster.GetNode(0)
err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false))
if err != nil {
t.Fatalf("could not toggle schema details to off: %v", err)
}
schema, err := cmd.API.SchemaDetails(context.Background())
if err != nil {
t.Fatalf("getting schema: %v", err)
}
for _, i := range schema {
for _, f := range i.Fields {
if f.Cardinality != nil {
t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality)
}
}
}
}
type mutexCheckIndex struct {
index *pilosa.Index
indexName string
@ -1357,7 +1344,9 @@ func TestVariousApiTranslateCalls(t *testing.T) {
if err != nil {
t.Fatalf("%v: could not create test index", err)
}
_, err = idx.CreateFieldIfNotExistsWithOptions("field", &pilosa.FieldOptions{Keys: false})
if _, err = idx.CreateFieldIfNotExistsWithOptions("field", &pilosa.FieldOptions{Keys: false}); err != nil {
t.Fatalf("creating field: %v", err)
}
t.Run("translateIndexDbOnNilIndex",
func(t *testing.T) {
err := api.TranslateIndexDB(context.Background(), "nonExistentIndex", 0, r)
@ -1415,3 +1404,289 @@ func TestVariousApiTranslateCalls(t *testing.T) {
*/
}
}
func TestAPI_CreateField(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(t, 3)
defer c.Close()
nodes := make([]*test.Command, 3)
for i := range nodes {
nodes[i] = c.GetNode(i)
}
if _, err := nodes[0].API.CreateIndex(ctx, "i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
eg, ctx := errgroup.WithContext(context.Background())
for _, n := range nodes {
node := n
eg.Go(func() error {
for i := 0; i < 10; i++ {
_, err := node.API.CreateField(ctx, "i", fmt.Sprintf("f%d", i))
if err != nil && !errors.Is(err, pilosa.ErrFieldExists) {
return err
}
}
return nil
})
}
err := eg.Wait()
if err != nil {
if errors.Is(err, pilosa.ErrFieldExists) {
t.Fatalf("conflict error: %v", err)
}
t.Fatalf("unexpected error: %T %v", err, err)
}
}
func TestAPI_RBFDebugInfo(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
)},
)
defer c.Close()
coord := c.GetPrimary()
if _, err := coord.API.CreateIndex(ctx, "i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if infos := coord.API.RBFDebugInfo(); infos == nil {
t.Fatal("expected info")
}
}
// makeUser makes an authnUserInfo from groups and a name and a secret key
func makeUser(t *testing.T, groups []authn.Group, name, secret string) *authn.UserInfo {
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = "42"
claims["name"] = name
secretKey, _ := hex.DecodeString(secret)
validToken, err := tkn.SignedString(secretKey)
if err != nil {
t.Fatalf("signing string %v", err)
}
validToken = "Bearer " + validToken
return &authn.UserInfo{
UserID: "fake" + name,
UserName: name,
Groups: groups,
Token: validToken,
Expiry: time.Time{},
}
}
func TestAuth_MultiNode(t *testing.T) {
// create permissions file
permissions := `
"user-groups":
"dca35310-ecda-4f23-86cd-876aee55906b":
"test": "read"
"dca35310-ecda-4f23-86cd-876aee55906f":
"test": "write"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
adminUser := makeUser(t, []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "adminGroup"}}, "admin", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
adminCtx := context.WithValue(
context.Background(),
"userinfo",
adminUser,
)
readUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "readGroup"}}, "reader", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
readCtx := context.WithValue(
context.Background(),
"userinfo",
readUser,
)
writeUser := makeUser(t, []authn.Group{{GroupID: "dca35310-ecda-4f23-86cd-876aee55906f", GroupName: "writeGroup"}}, "writer", "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEED")
writeCtx := context.WithValue(
context.Background(),
"userinfo",
writeUser,
)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
token, ok := r.Header["Authorization"]
if !ok || len(token) == 0 {
http.Error(w, "BAD REQUEST", http.StatusBadRequest)
return
}
g := []authn.Group{}
switch token[0] {
case adminUser.Token:
g = adminUser.Groups
case readUser.Token:
g = readUser.Groups
case writeUser.Token:
g = writeUser.Groups
}
if err := json.NewEncoder(w).Encode(authn.Groups{Groups: g}); err != nil {
t.Fatalf("unexpected error marshalling groups response: %v", err)
}
}))
// authentication on
auth := server.Auth{
Enable: true,
ClientId: "e9088663-eb08-41d7-8f65-efb5f54bbb71",
ClientSecret: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
AuthorizeURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize",
TokenURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token",
GroupEndpointURL: srv.URL,
RedirectBaseURL: "https://localhost:10101",
LogoutURL: "https://login.microsoftonline.com/common/oauth2/v2.0/logout",
Scopes: []string{"https://graph.microsoft.com/.default", "offline_access"},
SecretKey: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
PermissionsFile: writeTestFile(t, "permissions.yaml", permissions),
QueryLogPath: writeTestFile(t, "queryLog.log", ""),
}
config := server.NewConfig()
config.Auth = auth
// set up TLS certificates
localhostCert := `-----BEGIN CERTIFICATE-----
MIICEzCCAXygAwIBAgIQMIMChMLGrR+QvmQvpwAU6zANBgkqhkiG9w0BAQsFADAS
MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw
MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB
iQKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9SjY1bIw4
iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZBl2+XsDul
rKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQABo2gwZjAO
BgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUw
AwEB/zAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAAAAAAAAAA
AAAAATANBgkqhkiG9w0BAQsFAAOBgQCEcetwO59EWk7WiJsG4x8SY+UIAA+flUI9
tyC4lNhbcF2Idq9greZwbYCqTTTr2XiRNSMLCOjKyI7ukPoPjo16ocHj+P3vZGfs
h1fIw3cSS2OolhloGw/XM6RWPWtPAlGykKLciQrBru5NAPvCMsb/I1DAceTiotQM
fblo6RBxUQ==
-----END CERTIFICATE-----`
localhostKey := `-----BEGIN RSA PRIVATE KEY-----
MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9
SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZB
l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB
AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet
3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb
uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H
qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp
jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY
fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U
fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU
y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX
qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo
f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA==
-----END RSA PRIVATE KEY-----`
config.TLS.CertificateKeyPath = writeTestFile(t, "certKey.pem", localhostKey)
config.TLS.CertificatePath = writeTestFile(t, "cert.pem", localhostCert)
c := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&test.ModHasher{}),
),
server.OptCommandConfig(config),
},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node1"),
pilosa.OptServerClusterHasher(&test.ModHasher{}),
),
server.OptCommandConfig(config),
},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node2"),
pilosa.OptServerClusterHasher(&test.ModHasher{}),
),
server.OptCommandConfig(config),
},
)
defer c.Close()
primaryAPI := c.GetPrimary().API
// needs internal/cluster/message
indexName := "test"
_, err := primaryAPI.CreateIndex(adminCtx, indexName, pilosa.IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
// needs internal/translate/data
fieldName := "f"
_, err = primaryAPI.CreateField(adminCtx, indexName, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100))
if err != nil {
t.Fatalf("creating field: %v", err)
}
_, err = primaryAPI.Query(readCtx, &pilosa.QueryRequest{
Index: indexName,
Query: fmt.Sprintf(`Set(1, %s=1)`, fieldName),
})
if err == nil {
t.Fatalf("readCtx should not be able to set bits")
}
_, err = primaryAPI.Query(writeCtx, &pilosa.QueryRequest{
Index: indexName,
Query: fmt.Sprintf(`Set(1, %s=1)`, fieldName),
})
if err != nil {
t.Fatalf("writeCtx should be able to set bits: %v", err)
}
_, err = primaryAPI.Query(adminCtx, &pilosa.QueryRequest{
Index: indexName,
Query: fmt.Sprintf(`Set(1, %s=1)`, fieldName),
})
if err != nil {
t.Fatalf("adminCtx should be able to set bits: %v", err)
}
_, err = primaryAPI.Query(readCtx, &pilosa.QueryRequest{
Index: indexName,
Query: fmt.Sprintf(`Count(Row(%s=1))`, fieldName),
})
if err != nil {
t.Fatalf("readCtx should be able read: %v", err)
}
_, err = primaryAPI.Query(writeCtx, &pilosa.QueryRequest{
Index: indexName,
Query: fmt.Sprintf(`Count(Row(%s=1))`, fieldName),
})
if err != nil {
t.Fatalf("writeCtx should be able read: %v", err)
}
_, err = primaryAPI.Query(adminCtx, &pilosa.QueryRequest{
Index: indexName,
Query: fmt.Sprintf(`Count(Row(%s=1))`, fieldName),
})
if err != nil {
t.Fatalf("adminCtx should be able read: %v", err)
}
}
func writeTestFile(t *testing.T, filename, content string) string {
t.Helper()
fname := filepath.Join(t.TempDir(), filename)
f, err := os.Create(fname)
if err != nil {
t.Fatalf("could not create file %v with err %v", filename, err)
}
_, err = io.WriteString(f, content)
if err != nil {
t.Fatalf("could not write string %v", err)
}
defer f.Close()
return fname
}

2
audit.go
View file

@ -2,7 +2,7 @@
package pilosa
import (
"github.com/molecula/featurebase/v2/testhook"
"github.com/molecula/featurebase/v3/testhook"
)
var NewAuditor func() testhook.Auditor = NewNopAuditor

2
audit_internal_test.go
View file

@ -5,7 +5,7 @@ import (
"fmt"
"reflect"
"github.com/molecula/featurebase/v2/testhook"
"github.com/molecula/featurebase/v3/testhook"
)
// These audit hooks are desireable during testing, but not in

4
audit_test.go
View file

@ -6,8 +6,8 @@ import (
"os"
"reflect"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/testhook"
"github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/testhook"
)
// AuditLeaksOn is a global switch to turn on resource

25
auth/auth.go
View file

@ -1,25 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package auth
type Auth struct {
// Enable AuthZ/AuthN for featurebase server
Enable bool `toml:"enable"`
// Application/Client ID
ClientId string `toml:"client-id"`
// Client Secret
ClientSecret string `toml:"client-secret"`
// Authorize URL
AuthorizeURL string `toml:"authorize-url"`
// Token URL
TokenURL string `toml:"token-url"`
// Group Endpoint URL
GroupEndpointURL string `toml:"group-endpoint-url"`
// Scope URL
ScopeURL string `toml:"scope-url"`
}

307
authn/authenticate.go Normal file
View file

@ -0,0 +1,307 @@
// Copyright 2021 Molecula Corp. All rights reserved.
// Package authn handles authentication
package authn
import (
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/golang-jwt/jwt"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"github.com/molecula/featurebase/v3/logger"
"github.com/pkg/errors"
"golang.org/x/oauth2"
)
// cachedGroups is used to hold groups and when they were last cached
type cachedGroups struct {
cacheTime time.Time
groups []Group
}
// cacheToken is used to hold tokens and when they were added to the cache
type cachedToken struct {
cacheTime time.Time
token *oauth2.Token
}
// UserInfo holds the information about the user from the token
type UserInfo struct {
UserID string `json:"userid"`
UserName string `json:"username"`
Groups []Group `json:"groups"`
Expiry time.Time `json:"expiry"`
Token string `json:"token"`
}
// Group holds group information for an authenticated user
type Group struct {
GroupID string `json:"id"`
GroupName string `json:"displayName"`
}
// Groups holds a slice of Group for marshalling from JSON
type Groups struct {
Groups []Group `json:"value"`
}
// Auth holds state, configuration, and utilities needed for authentication.
type Auth struct {
logger logger.Logger
cookieName string
secretKey []byte
groupEndpoint string
logoutEndpoint string
fbURL string // fbURL is the domain featurebase is hosted on, used for post logout redirection
oAuthConfig *oauth2.Config
cacheTTL time.Duration // cacheTTL is used to determine if a cached item should be refreshed or not
tokenTTR time.Duration // tokenTTR (time to refresh) is used to determine if a token should be refreshed or not
tokenCache map[string]cachedToken // tokenCache is a map of accessToken -> *oauth2.Token which we can use to refresh the tokens
groupsCache map[string]cachedGroups // groupsCache is a map of accessToken -> group memberships
lastCacheClean time.Time // last cache clean is the time that the cache was last cleaned
}
// NewAuth instantiates and returns a new Auth struct
func NewAuth(logger logger.Logger, url string, scopes []string, authURL, tokenURL, groupEndpoint, logout, clientID, clientSecret, secretKey string) (auth *Auth, err error) {
auth = &Auth{
logger: logger,
cookieName: "molecula-chip",
groupEndpoint: groupEndpoint,
logoutEndpoint: logout,
fbURL: url,
oAuthConfig: &oauth2.Config{
RedirectURL: fmt.Sprintf("%s/redirect", url),
ClientID: clientID,
ClientSecret: clientSecret,
Scopes: scopes,
Endpoint: oauth2.Endpoint{
AuthURL: authURL,
TokenURL: tokenURL,
},
},
tokenCache: map[string]cachedToken{},
groupsCache: map[string]cachedGroups{},
cacheTTL: 10 * time.Minute,
tokenTTR: 7 * time.Minute,
lastCacheClean: time.Now(),
}
if auth.secretKey, err = decodeHex(secretKey); err != nil {
return nil, errors.Wrap(err, "decoding secret key")
}
return auth, nil
}
// SecretKey is a convenient function to get the SecretKey from an Auth struct
func (a Auth) SecretKey() []byte {
return a.secretKey
}
// Authenticate takes in a bearer token `bearer` and returns UserInfo from that token
// it is caller's responsibility to inform the user that the access token has been refreshed
func (a *Auth) Authenticate(ctx context.Context, bearer string) (*UserInfo, error) {
// clean up the cache every 30 minutes or so
if time.Now().Sub(a.lastCacheClean) >= 30*time.Minute {
a.cleanCache()
}
if tkn, ok := a.tokenCache[bearer]; ok && (tkn.token.Expiry.Sub(time.Now()) <= a.tokenTTR || !tkn.token.Valid()) {
// refresh the token
resp, err := http.PostForm(a.oAuthConfig.Endpoint.TokenURL,
url.Values{
"grant_type": {"refresh_token"},
"refresh_token": {tkn.token.RefreshToken},
"client_id": {a.oAuthConfig.ClientID},
"client_secret": {a.oAuthConfig.ClientSecret},
},
)
if err != nil {
return nil, errors.Wrap(err, "refreshing token")
}
defer resp.Body.Close()
var t oauth2.Token
if err := json.NewDecoder(resp.Body).Decode(&t); err != nil {
return nil, errors.Wrap(err, "decoding refreshed token")
}
// update the cache
delete(a.tokenCache, bearer)
delete(a.groupsCache, bearer)
bearer = t.AccessToken
a.tokenCache[bearer] = cachedToken{time.Now(), &t}
}
// NOTE: we are using ParseUnverified here because the IDP validates the
// token's signature when we get the user's groups, we just need to make
// sure it's not expired and is well-formed
token, _, err := new(jwt.Parser).ParseUnverified(bearer, &jwt.MapClaims{})
// well-formed-ness check
if token == nil || token.Claims == nil || err != nil {
return nil, fmt.Errorf("parsing bearer token: %v", err)
}
claims := *token.Claims.(*jwt.MapClaims)
// expiry check
if exp, ok := claims["exp"].(string); ok {
if expiry, err := strconv.ParseInt(exp, 10, 64); err != nil || expiry < time.Now().UTC().Unix() {
return nil, fmt.Errorf("token is expired")
}
}
userInfo := UserInfo{
UserID: claims["oid"].(string),
UserName: claims["name"].(string),
Token: bearer,
Groups: []Group{},
}
if userInfo.Groups, err = a.getGroups(bearer); err != nil {
return nil, errors.Wrap(err, "getting groups")
}
return &userInfo, nil
}
// cleanCache removes old items from our cache
func (a *Auth) cleanCache() {
for bearer, tkn := range a.tokenCache {
// if it's been more than 24 hours since the token was cached
if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
// remove it from our cache
delete(a.tokenCache, bearer)
}
}
for bearer, tkn := range a.groupsCache {
// if it's been more than 24 hours since the groups were cached
if time.Now().Sub(tkn.cacheTime) >= 24*time.Hour {
// remove it from our cache
delete(a.groupsCache, bearer)
}
}
a.lastCacheClean = time.Now()
}
// Login redirects a user to login to their configured oAuth authorize endpoint
func (a *Auth) Login(w http.ResponseWriter, r *http.Request) {
authURL := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
}
// Logout clears out the user's cookie, removes the token from our cache, and
// redirects user to IdP's logout endpoint
func (a *Auth) Logout(w http.ResponseWriter, r *http.Request) {
// remove the bearer token from a.tokenCache and a.groupsCache
if bearer, err := r.Cookie(a.cookieName); err == nil {
delete(a.tokenCache, bearer.Value)
delete(a.groupsCache, bearer.Value)
}
// clear cookie
http.SetCookie(w, &http.Cookie{
Name: a.cookieName,
Value: "",
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: time.Unix(0, 0),
})
http.Redirect(w, r, fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL), http.StatusTemporaryRedirect)
}
// Redirect handles the oAuth /redirect endpoint. It gets an access token and
// returns it to the user in the form of a cookie
func (a *Auth) Redirect(w http.ResponseWriter, r *http.Request) {
token, err := a.oAuthConfig.Exchange(r.Context(), r.FormValue("code"), oauth2.AccessTypeOffline)
if err != nil {
a.logger.Warnf("getting token from IdP: %+v", err)
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
a.tokenCache[token.AccessToken] = cachedToken{time.Now(), token}
a.SetCookie(w, token.AccessToken, token.Expiry)
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
}
// getGroups gets the group membership for a given token from configured IdP
func (a *Auth) getGroups(token string) ([]Group, error) {
var groups Groups
g, ok := a.groupsCache[token]
if ok && (time.Now().Sub(g.cacheTime) < a.cacheTTL) {
return g.groups, nil
}
req, err := http.NewRequest("GET", a.groupEndpoint, nil)
if err != nil {
return groups.Groups, errors.Wrap(err, "creating new request to group endpoint")
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
response, err := http.DefaultClient.Do(req)
if err != nil {
return groups.Groups, errors.Wrap(err, "getting group membership info")
}
defer response.Body.Close()
if err = json.NewDecoder(response.Body).Decode(&groups); err != nil {
return groups.Groups, errors.Wrap(err, "failed unmarshalling group membership response")
}
a.groupsCache[token] = cachedGroups{
cacheTime: time.Now(),
groups: groups.Groups,
}
return groups.Groups, nil
}
func (a *Auth) SetCookie(w http.ResponseWriter, token string, expiry time.Time) error {
http.SetCookie(w, &http.Cookie{
Name: a.cookieName,
Value: token,
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: expiry,
})
return nil
}
func (a *Auth) SetGRPCMetadata(ctx context.Context, md metadata.MD, token string) error {
cookies := []string{}
if c, ok := md["cookie"]; ok {
for _, cookie := range c {
if strings.HasPrefix(cookie, a.cookieName) {
cookie = a.cookieName + "=" + token
}
cookies = append(cookies, cookie)
}
}
md["cookie"] = cookies
return grpc.SetHeader(ctx, md)
}
func decodeHex(hexstr string) ([]byte, error) {
data, err := hex.DecodeString(hexstr)
if err != nil {
return nil, errors.Wrap(err, "decoding hex string to byte slice")
}
if len(data) != 32 {
return nil, fmt.Errorf("invalid key length")
}
return data, nil
}

592
authn/authenticate_internal_test.go Normal file
View file

@ -0,0 +1,592 @@
package authn
import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"reflect"
"strconv"
"strings"
"testing"
"time"
"github.com/golang-jwt/jwt"
"github.com/molecula/featurebase/v3/logger"
"golang.org/x/oauth2"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
func NewTestAuth(t *testing.T) *Auth {
t.Helper()
var (
ClientID = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
AuthorizeURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize"
TokenURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token"
GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true"
LogoutURL = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
Scopes = []string{"https://graph.microsoft.com/.default", "offline_access"}
Key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
)
a, err := NewAuth(
logger.NewStandardLogger(os.Stdout),
"http://localhost:10101/",
Scopes,
AuthorizeURL,
TokenURL,
GroupEndpointURL,
LogoutURL,
ClientID,
ClientSecret,
Key,
)
if err != nil {
t.Fatalf("building auth object%s", err)
}
return a
}
func TestAuth(t *testing.T) {
a := NewTestAuth(t)
t.Run("SetCookie", func(t *testing.T) {
w := httptest.NewRecorder()
err := a.SetCookie(w, "a cookie value", time.Now().Add(time.Hour))
if err != nil {
t.Fatalf("expected no errors, got: %v", err)
}
if w.Result().Cookies()[0].Value == "" {
t.Errorf("expected something, got empty string")
}
if got, want := w.Result().Cookies()[0].Path, "/"; got != want {
t.Fatalf("path=%s, want %s", got, want)
}
})
t.Run("SetGRPCMetadata", func(t *testing.T) {
md := metadata.MD{
"cookie": []string{a.cookieName + "=something"},
}
ctx := grpc.NewContextWithServerTransportStream(
metadata.NewIncomingContext(context.TODO(),
md,
),
NewServerTransportStream(),
)
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
err := a.SetGRPCMetadata(ctx, md, "this is a token!")
if err != nil {
t.Fatalf("expected no errors, got: %v", err)
}
md, ok = metadata.FromIncomingContext(ctx)
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
c, ok := md["cookie"]
if !ok {
t.Fatalf("expected ok, got: %v", ok)
}
var cookie string
for _, cookie = range c {
if strings.HasPrefix(cookie, a.cookieName) {
break
}
}
if exp, got := a.cookieName+"=this is a token!", cookie; got != exp {
t.Fatalf("expected '%v', got '%v'", exp, got)
}
})
t.Run("KeyLength", func(t *testing.T) {
_, err := NewAuth(
logger.NewStandardLogger(os.Stdout),
"http://localhost:10101/",
[]string{"https://graph.microsoft.com/.default", "offline_access"},
"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize",
"https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token",
"https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true",
"https://login.microsoftonline.com/common/oauth2/v2.0/logout",
"e9088663-eb08-41d7-8f65-efb5f54bbb71",
"DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
"DEADBEEFD",
)
if err == nil || !strings.Contains(err.Error(), "decoding secret key") {
t.Fatalf("expected error decoding secret key got: %v", err)
}
})
t.Run("GetSecretKey", func(t *testing.T) {
want, _ := hex.DecodeString("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
if got := a.SecretKey(); !bytes.Equal(got, want) {
t.Fatalf("expected %v, got %v", got, want)
}
})
}
func TestAuthenticate(t *testing.T) {
cases := []struct {
name string
uid string
uname string
exp int64
refresh bool
errOnRefresh bool
malformed bool
groups []Group
err error
}{
{
name: "GoodToken",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
},
{
name: "Malformed",
malformed: true,
err: fmt.Errorf("parsing bearer token: token contains an invalid number of segments"),
},
{
name: "ExpiredTokenNoRefresh",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
exp: -17764800,
err: fmt.Errorf("token is expired"),
},
{
name: "ExpiredTokenYesRefresh",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
refresh: true,
exp: -17764800,
},
{
name: "ExpiredTokenYesRefreshButError",
uid: "42",
uname: "A. Token",
groups: []Group{
{
GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
GroupName: "adminGroup",
},
},
refresh: true,
errOnRefresh: true,
exp: -17764800,
err: fmt.Errorf("decoding refreshed token: invalid character 'b' looking for beginning of value"),
},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
// setup the test
a := NewTestAuth(t)
token := ""
var err error
if !test.malformed {
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = test.uid
claims["name"] = test.uname
if test.exp != 0 {
claims["exp"] = strconv.Itoa(int(test.exp))
}
token, err = tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
} else {
token = "asdfasdfasdfasdF"
}
if len(test.groups) > 0 {
a.groupsCache[token] = cachedGroups{time.Now(), test.groups}
}
if test.refresh {
var srv *httptest.Server
if !test.errOnRefresh {
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = test.uid
claims["name"] = test.uname
expiry := strconv.Itoa(int(time.Now().Add(2 * time.Hour).Unix()))
claims["exp"] = expiry
fresh, err := tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
a.groupsCache[fresh] = cachedGroups{time.Now(), test.groups}
fmt.Fprintf(w, `{"access_token": "`+fresh+`", "refresh_token": "blah", "token_type": "bearer", "expires": `+expiry+` }`)
}))
} else {
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "bad", http.StatusInternalServerError)
}))
}
defer srv.Close()
a.oAuthConfig.Endpoint.TokenURL = srv.URL
a.tokenCache[token] = cachedToken{
time.Now(),
&oauth2.Token{
AccessToken: token,
RefreshToken: "blah",
Expiry: time.Unix(test.exp, 0),
},
}
}
// do the actual testing
uinfo, err := a.Authenticate(context.TODO(), token)
// okay this part kind of sucks bc we need to check errors and i
// dont want to write a whole new test for things that should have
// errors just to avoid this mess. errors.Is doesn't work either
if (test.err == nil && err != nil) || (test.err != nil && err == nil) {
t.Fatalf("expected %v, but got %v", test.err, err)
} else if test.err != nil && err != nil {
if test.err.Error() != err.Error() {
t.Fatalf("expected %v, but got %v", test.err, err)
} else {
return
}
}
if !reflect.DeepEqual(uinfo.Groups, test.groups) {
t.Fatalf("expected %v, got %v", test.groups, uinfo.Groups)
}
if !reflect.DeepEqual(uinfo.UserID, test.uid) {
t.Fatalf("expected %v, got %v", test.uid, uinfo.UserID)
}
if !reflect.DeepEqual(uinfo.UserName, test.uname) {
t.Fatalf("expected %v, got %v", test.uname, uinfo.UserName)
}
})
}
}
func TestAuthenticate_CleanCache(t *testing.T) {
// this deserves its own test bc it has gross setup required
t.Run("should clean", func(t *testing.T) {
a := NewTestAuth(t)
now := time.Now()
a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}}
a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}}
a.lastCacheClean = now.Add(-45 * time.Minute)
_, _ = a.Authenticate(context.TODO(), "this doesn't matter")
if a.lastCacheClean.Sub(now) <= time.Nanosecond {
t.Fatalf("cache should have been cleaned")
}
if _, ok := a.groupsCache["oldy"]; ok {
t.Errorf("oldy should have been deleted")
}
if _, ok := a.groupsCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
if _, ok := a.tokenCache["oldy"]; ok {
t.Errorf("oldy should have been deleted")
}
if _, ok := a.tokenCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
})
t.Run("shouldn't clean", func(t *testing.T) {
a := NewTestAuth(t)
now := time.Now()
a.groupsCache["oldy"] = cachedGroups{now.Add(-24 * time.Hour), []Group{}}
a.groupsCache["goldy"] = cachedGroups{now.Add(-4 * time.Hour), []Group{}}
a.tokenCache["oldy"] = cachedToken{now.Add(-24 * time.Hour), &oauth2.Token{}}
a.tokenCache["goldy"] = cachedToken{now.Add(-4 * time.Hour), &oauth2.Token{}}
a.lastCacheClean = now
_, _ = a.Authenticate(context.TODO(), "this doesn't matter")
if a.lastCacheClean.Sub(now) >= time.Nanosecond {
t.Fatalf("cache should not have been cleaned")
}
if _, ok := a.groupsCache["oldy"]; !ok {
t.Errorf("oldy should not have been deleted")
}
if _, ok := a.groupsCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
if _, ok := a.tokenCache["oldy"]; !ok {
t.Errorf("oldy should not have been deleted")
}
if _, ok := a.tokenCache["goldy"]; !ok {
t.Errorf("goldy should not have been deleted")
}
})
}
func TestGetGroups(t *testing.T) {
a := NewTestAuth(t)
a.groupsCache = map[string]cachedGroups{
"the world is changed": {
cacheTime: time.Now(),
groups: []Group{
{
GroupID: "i feel it in the water",
GroupName: "i feel it in the earth",
},
},
},
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := json.Marshal(
Groups{
Groups: []Group{
{
GroupID: "much that once was is lost",
GroupName: "for none now live who remember it",
},
},
},
)
if err != nil {
t.Fatalf("unexpected error marshalling groups response: %v", err)
}
fmt.Fprintf(w, "%s", body)
}))
a.groupEndpoint = srv.URL
for name, test := range map[string]struct {
token string
groups []Group
}{
"InCache": {
token: "the world is changed",
groups: []Group{
{
GroupID: "i feel it in the water",
GroupName: "i feel it in the earth",
},
},
},
"NotInCache": {
token: "i smell it in the air",
groups: []Group{
{
GroupID: "much that once was is lost",
GroupName: "for none now live who remember it",
},
},
},
} {
t.Run(name, func(t *testing.T) {
if got, err := a.getGroups(test.token); err != nil || !reflect.DeepEqual(got, test.groups) {
t.Errorf("expected %v, nil, got %v, %v", test.groups, got, err)
}
})
}
}
func TestDecodeHex(t *testing.T) {
t.Run("cantDecode", func(t *testing.T) {
_, err := decodeHex("gggg")
if err == nil {
t.Fatalf("expected err cannot decode slice, got nil")
}
})
t.Run("tooSmall", func(t *testing.T) {
_, err := decodeHex("DEADBEEF")
if err == nil {
t.Fatalf("expected err wrong length, got nil")
}
})
t.Run("tooBig", func(t *testing.T) {
_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
if err == nil {
t.Fatalf("expected err wrong length, got nil")
}
})
t.Run("justRight", func(t *testing.T) {
_, err := decodeHex("DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF")
if err != nil {
t.Fatalf("expected nil, got %v", err)
}
})
}
func TestHandlers(t *testing.T) {
a := NewTestAuth(t)
t.Run("login", func(t *testing.T) {
req := httptest.NewRequest("GET", "/login", nil)
w := httptest.NewRecorder()
a.Login(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusTemporaryRedirect {
t.Fatalf("expected redirect, got %v", resp.StatusCode)
}
redirect := a.oAuthConfig.AuthCodeURL(a.oAuthConfig.Endpoint.AuthURL)
if got, err := resp.Location(); err != nil || got.String() != redirect {
t.Fatalf("expected %v, got %v", redirect, got.Path)
}
})
t.Run("logout", func(t *testing.T) {
req := httptest.NewRequest("GET", "/logout", nil)
w := httptest.NewRecorder()
req.AddCookie(
&http.Cookie{
Name: a.cookieName,
Value: "test",
Path: "/",
Secure: true,
HttpOnly: true,
SameSite: http.SameSiteStrictMode,
Expires: time.Unix(3000000, 0),
},
)
a.groupsCache["test"] = cachedGroups{}
a.tokenCache["test"] = cachedToken{time.Now(), &oauth2.Token{}}
a.Logout(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusTemporaryRedirect {
t.Fatalf("expected redirect, got %v", resp.StatusCode)
}
redirect := fmt.Sprintf("%s?post_logout_redirect_uri=%s/", a.logoutEndpoint, a.fbURL)
if got, err := resp.Location(); err != nil || got.String() != redirect {
t.Fatalf("expected %v, got %v", redirect, got.Path)
}
for _, c := range resp.Cookies() {
if c.Name == a.cookieName {
if c.Value != "" {
t.Fatalf("cookie not set to empty value!")
}
want := time.Unix(0, 0).Unix()
got := c.Expires.Unix()
if want != got {
t.Fatalf("expected %v, got %v", want, got)
}
break
}
}
if _, ok := a.groupsCache["test"]; ok {
t.Fatalf("groups not deleted!")
}
if _, ok := a.tokenCache["test"]; ok {
t.Fatalf("token not deleted!")
}
})
t.Run("redirectGood", func(t *testing.T) {
req := httptest.NewRequest("GET", "/redirect", nil)
w := httptest.NewRecorder()
tkn := jwt.New(jwt.SigningMethodHS256)
claims := tkn.Claims.(jwt.MapClaims)
claims["oid"] = "user id"
claims["name"] = "user name"
expiresIn := 2 * time.Hour
exp := time.Now().Add(expiresIn)
expiry := strconv.Itoa(int(exp.Unix()))
claims["exp"] = expiry
fresh, err := tkn.SignedString(a.SecretKey())
if err != nil {
t.Fatalf("unexpected error when signing token %v", err)
}
freshToken := oauth2.Token{
AccessToken: fresh,
RefreshToken: "blah",
Expiry: exp,
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body := `{"access_token": "` + fresh + `", "refresh_token": "blah", "expires_in": "` + strconv.Itoa(int(expiresIn.Seconds())) + `"}`
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
w.Write([]byte(body))
}))
a.oAuthConfig.Endpoint.TokenURL = srv.URL
a.Redirect(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusTemporaryRedirect {
t.Fatalf("expected redirect, got %v", resp.StatusCode)
}
if got, err := resp.Location(); err != nil || got.String() != "/" {
t.Fatalf("expected %v, got %v", "/", got.Path)
}
cachedToken := a.tokenCache[fresh].token
if cachedToken.AccessToken != freshToken.AccessToken {
t.Fatalf("expected %v, got %v", freshToken.AccessToken, cachedToken.AccessToken)
}
if cachedToken.RefreshToken != freshToken.RefreshToken {
t.Fatalf("expected %v, got %v", freshToken.RefreshToken, cachedToken.RefreshToken)
}
if cachedToken.Expiry.Sub(freshToken.Expiry) > time.Second {
t.Fatalf("expected %v, got %v", freshToken.Expiry, cachedToken.Expiry)
}
})
t.Run("redirectBad", func(t *testing.T) {
req := httptest.NewRequest("GET", "/redirect", nil)
w := httptest.NewRecorder()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Server Error", http.StatusInternalServerError)
}))
a.oAuthConfig.Endpoint.TokenURL = srv.URL
a.Redirect(w, req)
resp := w.Result()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected BadRequest, got %v", resp.StatusCode)
}
})
}
// This type is used for mocking ServerTransportStreams in tests
type ServerTransportStream struct {
md metadata.MD
method string
}
func NewServerTransportStream() *ServerTransportStream {
return &ServerTransportStream{
md: metadata.MD{},
method: "test",
}
}
func (s *ServerTransportStream) Method() string {
return s.method
}
func (s *ServerTransportStream) SetHeader(md metadata.MD) error {
s.md = md
return nil
}
func (s *ServerTransportStream) SendHeader(md metadata.MD) error {
_ = md
return nil
}
func (s *ServerTransportStream) SetTrailer(md metadata.MD) error {
_ = md
return nil
}

142
authz/authorization.go Normal file
View file

@ -0,0 +1,142 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package authz
import (
"fmt"
"io"
"io/ioutil"
"github.com/molecula/featurebase/v3/authn"
"gopkg.in/yaml.v2"
)
type GroupPermissions struct {
Permissions map[string]map[string]Permission `yaml:"user-groups"`
Admin string `yaml:"admin"`
}
type Permission string
const (
None Permission = ""
Read Permission = "read"
Write Permission = "write"
Admin Permission = "admin"
)
// Satisfies returns whether `p` satisfies the permissions required by `b`
func (p Permission) Satisfies(b Permission) bool {
switch p {
case "":
return b == ""
case "read":
return b == "" || b == "read"
case "write":
return b == "" || b == "read" || b == "write"
case "admin":
return b == "" || b == "read" || b == "write" || b == "admin"
}
return false
}
func (p *GroupPermissions) ReadPermissionsFile(permsFile io.Reader) (err error) {
permsData, err := ioutil.ReadAll(permsFile)
if err != nil {
return fmt.Errorf("reading permissions failed with error: %s", err)
}
err = yaml.UnmarshalStrict(permsData, &p)
if err != nil {
return fmt.Errorf("unmarshalling permissions failed with error: %s", err)
}
return
}
func (p *GroupPermissions) GetPermissions(user *authn.UserInfo, index string) (permission Permission, errors error) {
groups := user.Groups
if admin := p.IsAdmin(groups); admin {
return Admin, nil
}
allPermissions := map[Permission]bool{
Write: false,
Read: false,
}
if len(groups) == 0 {
return None, fmt.Errorf("user is not part of any groups in identity provider")
}
var groupsDenied []string
for _, group := range groups {
if _, ok := p.Permissions[group.GroupID]; ok {
if perm, ok := p.Permissions[group.GroupID][index]; ok {
allPermissions[perm] = true
} else {
return None, fmt.Errorf("user %s does not have permission to index %s", user.UserID, index)
}
} else {
groupsDenied = append(groupsDenied, group.GroupID)
}
}
if len(groupsDenied) == len(groups) {
return None, fmt.Errorf("group(s) %s does not have permission to FeatureBase", groupsDenied)
}
if allPermissions[Write] {
return Write, nil
} else if allPermissions[Read] {
return Read, nil
} else {
return None, fmt.Errorf("no permissions found")
}
}
func (p *GroupPermissions) IsAdmin(groups []authn.Group) bool {
for _, group := range groups {
if p.Admin == group.GroupID {
return true
}
}
return false
}
func (p *GroupPermissions) GetAuthorizedIndexList(groups []authn.Group, desiredPermission Permission) (indexList []string) {
// if user is admin, find all indexes in permissions file and return them
if p.IsAdmin(groups) {
for groupId := range p.Permissions {
for index := range p.Permissions[groupId] {
indexList = append(indexList, index)
}
}
return indexList
}
for _, group := range groups {
if _, ok := p.Permissions[group.GroupID]; ok {
for index, permission := range p.Permissions[group.GroupID] {
if permission.Satisfies(desiredPermission) {
indexList = append(indexList, index)
}
}
}
}
return indexList
}

316
authz/authorization_test.go Normal file
View file

@ -0,0 +1,316 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package authz_test
import (
"fmt"
"reflect"
"sort"
"strings"
"testing"
"github.com/molecula/featurebase/v3/authn"
"github.com/molecula/featurebase/v3/authz"
)
func TestAuth_ReadPermissionsFile(t *testing.T) {
singleInput := `user-groups:
"dca35310-ecda-4f23-86cd-876aee55906b":
"test": "read"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
multiInput := `user-groups:
"dca35310-ecda-4f23-86cd-876aee55906b":
"test": "read"
"test2": "write"
"dca35310-ecda-4f23-86cd-876aee559900":
"test": "write"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
singlePermission := authz.GroupPermissions{
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read},
},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
multiPermission := authz.GroupPermissions{
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Read, "test2": authz.Write},
"dca35310-ecda-4f23-86cd-876aee559900": {"test": authz.Write}},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
tests := []struct {
input string
output authz.GroupPermissions
}{
{singleInput, singlePermission},
{multiInput, multiPermission},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
permFile := strings.NewReader(test.input)
var p authz.GroupPermissions
err := p.ReadPermissionsFile(permFile)
if err != nil {
t.Fatalf("readPermissionsFile error: %s", err)
}
if !reflect.DeepEqual(p, test.output) {
t.Fatalf("expected output %s, but got %s", test.output, p)
}
},
)
}
}
func TestAuth_GetPermissions(t *testing.T) {
// initializes different example of permissions file in yaml
permissions1 := `"user-groups":
"dca35310-ecda-4f23-86cd-876aee55906b":
"test": "read"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
permissions2 := `"user-groups":
"dca35310-ecda-4f23-86cd-876aee559900":
"test": "write"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
permissions3 := `"user-groups":
"dca35310-ecda-4f23-86cd-876aee55906b":
"test": "write"
"test2": "read"
"dca35310-ecda-4f23-86cd-876aee559900":
"test": "read"
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
permissions4 := `"user-groups":
"dca35310-ecda-4f23-86cd-876aee559900":
"test": ""
admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe"`
// initializes groups that are returned from identity provider
groupName := "name"
groupsList1 := []authn.Group{}
groupsList2 := []authn.Group{{
GroupID: "fake-group",
GroupName: groupName}}
groupsList3 := []authn.Group{
{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: groupName},
{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: groupName},
}
groupsList4 := []authn.Group{{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: groupName}}
tests := []struct {
yamlData string
groups []authn.Group
index string
userAccess authz.Permission
err string
}{
{
permissions1,
groupsList1,
"test",
authz.None,
"user is not part of any groups in identity provider",
},
{
permissions1,
groupsList3,
"test1",
authz.None,
"does not have permission to index",
},
{
permissions2,
groupsList2,
"test",
authz.None,
"does not have permission to FeatureBase",
},
{
permissions1,
groupsList3,
"test",
authz.Read,
"",
},
{
permissions2,
groupsList3,
"test",
authz.Write,
"",
},
{
permissions3,
groupsList4,
"test",
authz.Admin,
"",
},
{
permissions4,
groupsList3,
"test",
authz.None,
"no permissions found",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
permFile := strings.NewReader(test.yamlData)
var p authz.GroupPermissions
if err := p.ReadPermissionsFile(permFile); err != nil {
t.Errorf("Error: %s", err)
}
p1, err := p.GetPermissions(&authn.UserInfo{Groups: test.groups}, test.index)
if p1 != test.userAccess {
t.Errorf("expected permission to be %s, but got %s", test.userAccess, p1)
}
if err != nil {
if !strings.Contains(err.Error(), test.err) {
t.Errorf("expected error to contain %s, but got %s", test.err, err.Error())
}
}
})
}
}
func TestAuth_IsAdmin(t *testing.T) {
group1 := []authn.Group{
{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
}
group2 := []authn.Group{
{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
}
groupPermissions := authz.GroupPermissions{
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {"test": authz.Write},
},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
tests := []struct {
groups []authn.Group
groupPermissions authz.GroupPermissions
output bool
}{
{
group1, groupPermissions, true,
},
{
group2, groupPermissions, false,
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
p := test.groupPermissions
resp := p.IsAdmin(test.groups)
if resp != test.output {
t.Errorf("expected %t, but got %t", test.output, resp)
}
})
}
}
func TestAuth_GetAuthorizedIndexList(t *testing.T) {
group1 := []authn.Group{
{GroupID: "dca35310-ecda-4f23-86cd-876aee55906b", GroupName: "group-name"},
}
group2 := []authn.Group{
{GroupID: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe", GroupName: "admin-group"},
}
group3 := []authn.Group{
{GroupID: "dca35310-ecda-4f23-86cd-876aee559900", GroupName: "group-name"},
}
p := authz.GroupPermissions{
Permissions: map[string]map[string]authz.Permission{
"dca35310-ecda-4f23-86cd-876aee55906b": {
"test1": authz.Read,
"test2": authz.Write,
},
"dca35310-ecda-4f23-86cd-876aee559900": {
"test3": authz.Read,
},
},
Admin: "ac97c9e2-346b-42a2-b6da-18bcb61a32fe",
}
tests := []struct {
groups []authn.Group
permission authz.Permission
output []string
}{
{
group1,
authz.Read,
[]string{"test1", "test2"},
},
{
group1,
authz.Write,
[]string{"test2"},
},
{
group3,
authz.Write,
nil,
},
{
group2,
authz.Read,
[]string{"test1", "test2", "test3"},
},
{
group2,
authz.Write,
[]string{"test1", "test2", "test3"},
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
indexList := p.GetAuthorizedIndexList(test.groups, test.permission)
sort.Strings(indexList)
if !reflect.DeepEqual(indexList, test.output) {
t.Errorf("expected %s, but got %s", test.output, indexList)
}
})
}
}

181
boltdb/translate.go
View file

@ -12,7 +12,8 @@ import (
"sync"
"time"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
@ -32,6 +33,8 @@ var (
bucketKeys = []byte("keys")
bucketIDs = []byte("ids")
bucketFree = []byte("free")
freeKey = []byte("free")
)
const (
@ -119,6 +122,8 @@ func (s *TranslateStore) Open() (err error) {
return err
} else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil {
return err
} else if _, err := tx.CreateBucketIfNotExists(bucketFree); err != nil {
return err
}
return nil
}); err != nil {
@ -230,14 +235,26 @@ func (s *TranslateStore) CreateKeys(keys ...string) (map[string]uint64, error) {
if idBucket == nil {
return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs)
}
freeBucket := tx.Bucket(bucketFree)
if freeBucket == nil {
return errors.Errorf(errFmtTranslateBucketNotFound, bucketFree)
}
puts := 0
// we create a freeIDGetter to reduce marshalling
getter := newFreeIDGetter(freeBucket)
defer getter.Close()
for idx, key := range keys {
id, boltKey := findIDByKey(keyBucket, key)
if id != 0 {
result[key] = id
continue
}
id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
// see if we can re-use any IDs first
if id = getter.GetFreeID(); id == 0 {
id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN)
}
idBytes := idScratch[puts*8 : puts*8+8]
binary.BigEndian.PutUint64(idBytes, id)
puts++
@ -498,6 +515,88 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error {
}
}
type boltWrapper struct {
tx *bolt.Tx
db *bolt.DB
}
func (w *boltWrapper) Commit() error {
if w.tx != nil {
return w.tx.Commit()
}
return nil
}
func (w *boltWrapper) Rollback() {
if w.tx != nil {
w.tx.Rollback()
}
}
func (s *TranslateStore) FreeIDs() (*roaring.Bitmap, error) {
result := roaring.NewBitmap()
err := s.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(bucketFree)
if bkt == nil {
return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys)
}
b := bkt.Get(freeKey)
err := result.UnmarshalBinary(b)
if err != nil {
return err
}
return nil
})
return result, err
}
func (s *TranslateStore) MergeFree(tx *bolt.Tx, newIDs *roaring.Bitmap) error {
bkt := tx.Bucket(bucketFree)
b := bkt.Get(freeKey)
buf := new(bytes.Buffer)
if b != nil { //if existing combine with newIDs
before := roaring.NewBitmap()
err := before.UnmarshalBinary(b)
if err != nil {
return err
}
final := newIDs.Union(before)
_, err = final.WriteTo(buf)
if err != nil {
return err
}
} else {
newIDs.WriteTo(buf)
}
return bkt.Put(freeKey, buf.Bytes())
}
// Delete removes the lookeup pairs in order to make avialble for reuse but doesn't commit the
// transaction for that is tied to the associated rbf transaction being successful
func (s *TranslateStore) Delete(records *roaring.Bitmap) (pilosa.Commitor, error) {
tx, err := s.db.Begin(true)
if err != nil {
return nil, err
}
keyBucket := tx.Bucket(bucketKeys)
idBucket := tx.Bucket(bucketIDs)
ids := records.Slice()
for i := range ids {
id := u64tob(ids[i])
boltKey := idBucket.Get(id)
err = keyBucket.Delete(boltKey)
if err != nil {
tx.Rollback()
return &boltWrapper{}, err
}
err = idBucket.Delete(id)
if err != nil {
tx.Rollback()
return &boltWrapper{}, err
}
}
return &boltWrapper{tx: tx}, s.MergeFree(tx, records)
}
// emptyKey is a sentinel byte slice which stands for "" as a key.
var emptyKey = []byte{
0x00, 0x00, 0x00,
@ -521,6 +620,84 @@ func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) {
return 0, boltKey
}
// freeIDGetter reduces the amount of marshaling required to get multiple ids
type freeIDGetter struct {
freeBucket *bolt.Bucket
b *roaring.Bitmap
changed bool
}
// newFreeIDGetter initializes a new freeIDGetter. If at any point there is a
// failure, it returns an error.
//
// NOTE: For changes to be persisted to the bucket, you must call
// (*freeIDGetter).Close()
func newFreeIDGetter(freeBucket *bolt.Bucket) *freeIDGetter {
g := &freeIDGetter{
freeBucket: freeBucket,
}
// we ignore this value because it's okay if we dont have a bitmap just yet
_ = g.getBitmap()
return g
}
func (g *freeIDGetter) getBitmap() bool {
if g.b == nil {
// get the bitmap from freeBucket
value := g.freeBucket.Get(freeKey)
if value == nil {
return false
}
// turn the value into a bitmap
b := roaring.NewBitmap()
if err := b.UnmarshalBinary(value); err != nil {
return false
}
g.b = b
}
return true
}
// GetFreeID tries to get a free ID from the free id bucket. If at any point it
// fails to do so, it returns a 0. Otherwise, it returns the first free ID in the
// bucket
func (g *freeIDGetter) GetFreeID() (id uint64) {
if !g.getBitmap() {
return 0
}
// get the first free id
id, ok := g.b.Min()
if !ok {
return 0
}
// remove that id from the free id bitmap
if changed, err := g.b.RemoveN(id); changed == 0 || err != nil {
return 0
} else {
g.changed = true
}
return id
}
// Close persists any changes to the bitmap back to the bucket and then nils the
// references for safety.
func (g *freeIDGetter) Close() error {
if g.changed {
// convert bitmap to binary
buf, err := g.b.MarshalBinary()
if err != nil {
return errors.Wrap(err, "closing free ID Getter")
}
// put updated bitmap back into the freeBucket
if err := g.freeBucket.Put(freeKey, buf); err != nil {
return errors.Wrap(err, "closing free ID Getter")
}
}
g.b = nil
g.freeBucket = nil
return nil
}
func findKeyByID(bkt *bolt.Bucket, id uint64) string {
boltKey := bkt.Get(u64tob(id))
if bytes.Equal(boltKey, emptyKey) {

107
boltdb/translate_internal_test.go Normal file
View file

@ -0,0 +1,107 @@
package boltdb
import (
"path/filepath"
"testing"
"github.com/molecula/featurebase/v3/roaring"
bolt "go.etcd.io/bbolt"
)
func TestGetFreeID(t *testing.T) {
boltDir := t.TempDir()
db, err := bolt.Open(filepath.Join(boltDir, "testDB"), 0600, nil)
if err != nil {
t.Fatalf("unexpected error opening test boltdb: %v", err)
}
defer db.Close()
makeTestBucket := func(tx *bolt.Tx, b *roaring.Bitmap) *bolt.Bucket {
if b == nil {
t.Fatalf("unexpected nil bitmap")
}
free, err := tx.CreateBucketIfNotExists(bucketFree)
if err != nil {
t.Fatalf("unexpected error making freeBucket: %v", err)
}
buf, err := b.MarshalBinary()
if err != nil {
t.Fatalf("unexpected error marshaling bitmap (%v) to binary: %v", b, err)
}
if err := free.Put(freeKey, buf); err != nil {
t.Fatalf("unexpected error adding data (%v) to freeBucket: %v", b, err)
}
return free
}
for name, test := range map[string]struct {
bits *roaring.Bitmap
want uint64
}{
"bucket is there, but nobody's home": {
bits: roaring.NewBitmap(),
want: 0,
},
"good bucket": {
bits: roaring.NewBitmap(1, 2, 34, 55, 9000),
want: 1,
},
} {
t.Run(name, func(t *testing.T) {
tx, err := db.Begin(true)
if err != nil {
t.Fatalf("unexpected error starting bolt transaction: %v", err)
}
defer tx.Rollback()
freeBucket := makeTestBucket(tx, test.bits)
getter := newFreeIDGetter(freeBucket)
defer getter.Close()
if got := getter.GetFreeID(); got != test.want {
t.Fatalf("expected %v got %v", test.want, got)
}
})
}
t.Run("CorrectOrdering", func(t *testing.T) {
tx, err := db.Begin(true)
if err != nil {
t.Fatalf("unexpected error starting bolt transaction: %v", err)
}
defer tx.Rollback()
bucket := makeTestBucket(tx, roaring.NewBitmap(1, 34, 2, 55, 9000))
getter := newFreeIDGetter(bucket)
defer getter.Close()
for _, want := range []uint64{1, 2, 34, 55, 9000} {
if got := getter.GetFreeID(); got != want {
t.Fatalf("expected %v got %v", want, got)
}
}
if got := getter.GetFreeID(); got != 0 {
t.Fatalf("expected 0 got %v", got)
}
})
t.Run("NotABitmap", func(t *testing.T) {
tx, err := db.Begin(true)
if err != nil {
t.Fatalf("unexpected error starting bolt transaction: %v", err)
}
defer tx.Rollback()
free, err := tx.CreateBucketIfNotExists(bucketFree)
if err != nil {
t.Fatalf("unexpected error making freeBucket: %v", err)
}
if err := free.Put(freeKey, []byte("this isn't right!")); err != nil {
t.Fatalf("unexpected error adding data to freeBucket: %v", err)
}
getter := newFreeIDGetter(free)
defer getter.Close()
if got := getter.GetFreeID(); got != 0 {
t.Fatalf("expected 0 got %v", got)
}
})
}

59
boltdb/translate_test.go
View file

@ -10,10 +10,11 @@ import (
"testing"
"time"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/boltdb"
"github.com/molecula/featurebase/v2/testhook"
"github.com/molecula/featurebase/v2/topology"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/testhook"
"github.com/molecula/featurebase/v3/topology"
)
//var vv = pilosa.VV
@ -385,7 +386,53 @@ func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
s.Path = f.Name()
return s
}
func TestTranslateStore_Delete(t *testing.T) {
s := MustOpenNewTranslateStore(t)
defer MustCloseTranslateStore(s)
// Setup initial keys.
ids, err := s.CreateKeys("foo", "bar", "deleteme")
if err != nil {
t.Fatal(err)
}
records := roaring.NewBitmap(ids["deleteme"])
c, err := s.Delete(records)
if err != nil {
t.Fatal(err)
}
if err = c.Commit(); err != nil {
t.Fatal(err)
}
r, e := s.FreeIDs()
if e != nil {
t.Fatal(err)
}
freeids := r.Slice()
if len(freeids) == 0 {
t.Fatalf("expected to have free id")
}
if freeids[0] != ids["deleteme"] {
t.Fatalf("expected [%v] and got %v", ids["deleteme"], freeids[0])
}
records2 := roaring.NewBitmap(ids["foo"])
c, err = s.Delete(records2)
if err != nil {
t.Fatal(err)
}
if err = c.Commit(); err != nil {
t.Fatal(err)
}
r, e = s.FreeIDs()
if e != nil {
t.Fatal(err)
}
freeids = r.Slice()
if len(freeids) != 2 {
t.Fatalf("expected to have 2 free ids")
}
}
func TestTranslateStore_ReadWrite(t *testing.T) {
t.Run("WriteTo_ReadFrom", func(t *testing.T) {
s := MustOpenNewTranslateStore(t)
@ -408,7 +455,7 @@ func TestTranslateStore_ReadWrite(t *testing.T) {
// Put the contents of the store into a buffer.
buf := bytes.NewBuffer(nil)
expN := int64(32768)
expN := s.Size()
// After this, the buffer should contain batch0.
if n, err := s.WriteTo(buf); err != nil {
@ -458,7 +505,7 @@ func TestTranslateStore_ReadWrite(t *testing.T) {
func MustOpenNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
s := MustNewTranslateStore(tb)
if err := s.Open(); err != nil {
panic(err)
tb.Fatalf("opening s: %v", err)
}
return s
}

2
broadcast.go
View file

@ -4,7 +4,7 @@ package pilosa
import (
"fmt"
"github.com/molecula/featurebase/v2/topology"
"github.com/molecula/featurebase/v3/topology"
"github.com/pkg/errors"
)

2
bsi.go
View file

@ -4,7 +4,7 @@ package pilosa
import (
"math/bits"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v3/roaring"
)
// bsiData contains BSI-structured data.

45
cache.go
View file

@ -10,9 +10,9 @@ import (
"sync"
"time"
"github.com/molecula/featurebase/v2/lru"
pb "github.com/molecula/featurebase/v2/proto"
"github.com/molecula/featurebase/v2/stats"
"github.com/molecula/featurebase/v3/lru"
pb "github.com/molecula/featurebase/v3/proto"
"github.com/molecula/featurebase/v3/stats"
"github.com/pkg/errors"
)
@ -194,6 +194,14 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
}
c.entries[id] = n
// FB-1206: Periodically invalidate the cache when we are bulk loading
// as this can take up an upbounded amount of memory. This is especially
// true when restoring shards as all rows will be added.
if len(c.entries) > int(2*c.maxEntries) {
c.stats.Count(MetricRecalculateCache, 1, 1.0)
c.recalculate()
}
}
// Get returns a count for a given id.
@ -567,37 +575,6 @@ func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
// simpleCache implements a bitmap Rowcache.
// it is meant to be a short-lived cache for cases where writes are continuing to access
// the same row within a short time frame (i.e. good for write-heavy loads)
// A read-heavy use case would cause the cache to get bigger, potentially causing the
// node to run out of memory.
type simpleCache struct {
cache map[uint64]*Row
}
// Fetch retrieves the bitmap at the id in the cache.
func (s *simpleCache) Fetch(id uint64) (*Row, bool) {
m, ok := s.cache[id]
return m, ok
}
func newSimpleCache() *simpleCache {
return &simpleCache{
cache: make(map[uint64]*Row),
}
}
// Add adds the bitmap to the cache, keyed on the id. A nil row means
// deleting the row from the cache.
func (s *simpleCache) Add(id uint64, b *Row) {
if b != nil {
s.cache[id] = b
} else {
delete(s.cache, id)
}
}
// nopCache represents a no-op Cache implementation.
type nopCache struct {
stats stats.StatsClient

14
cache_test.go
View file

@ -5,7 +5,7 @@ import (
"reflect"
"testing"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v3"
)
// Ensure cache stays constrained to its configured size.
@ -70,3 +70,15 @@ func TestCache_Rank_Dirty(t *testing.T) {
t.Fatalf("wrote %v but got %v", expect, got)
}
}
func TestCache_Rank_BulkAdd(t *testing.T) {
const cacheSize = 10
cache := pilosa.NewRankCache(uint32(cacheSize))
for i := uint64(0); i < 1000; i++ {
cache.BulkAdd(i, i)
if n := cache.Len(); n > cacheSize*2 {
t.Fatalf("entry count exceed 2x cache size: %d", n)
}
}
}

11
catcher.go
View file

@ -2,9 +2,9 @@
package pilosa
import (
"github.com/molecula/featurebase/v2/roaring"
txkey "github.com/molecula/featurebase/v2/short_txkey"
"github.com/molecula/featurebase/v2/vprint"
"github.com/molecula/featurebase/v3/roaring"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/vprint"
)
// catcher is useful to report error locations with a
@ -26,6 +26,11 @@ func init() {
var _ Tx = (*catcherTx)(nil)
func (c *catcherTx) RemoveChannel(index, field, view string, shard uint64, a chan uint64, resChan chan countResults) {
c.b.RemoveChannel(index, field, view, shard, a, resChan)
return
}
func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
return c.b.NewTxIterator(index, field, view, shard)
}

289
client.go
View file

@ -1,289 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"context"
"io"
"time"
"github.com/molecula/featurebase/v2/ingest"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/topology"
)
// Bit represents the intersection of a row and a column. It can be specified by
// integer ids or string keys.
type Bit struct {
RowID uint64
ColumnID uint64
RowKey string
ColumnKey string
Timestamp int64
}
// FieldValue represents the value for a column within a
// range-encoded field.
type FieldValue struct {
ColumnID uint64
ColumnKey string
Value int64
}
// InternalClient should be implemented by any struct that enables any transport between nodes
// TODO: Refactor
// Note from Travis: Typically an interface containing more than two or three methods is an indication that
// something hasn't been architected correctly.
// While I understand that putting the entire Client behind an interface might require this many methods,
// I don't want to let it go unquestioned.
// Another note from Travis: I think we eventually want to unify `InternalClient` with
// the `github.com/molecula/featurebase/v2/client` client.
// Doing that may obviate the need to refactor this.
type InternalClient interface {
InternalQueryClient
AvailableShards(ctx context.Context, indexName string) ([]uint64, error)
MaxShardByIndex(ctx context.Context) (map[string]uint64, error)
Schema(ctx context.Context) ([]*IndexInfo, error)
PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error)
PartitionNodes(ctx context.Context, partitionID int) ([]*topology.Node, error)
Nodes(ctx context.Context) ([]*topology.Node, error)
Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error)
Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
EnsureField(ctx context.Context, indexName string, fieldName string) error
EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error
ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error
CreateField(ctx context.Context, index, field string) error
CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error
FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error)
BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error)
SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error)
RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error)
ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error)
MutexCheck(ctx context.Context, uri *pnet.URI, index string, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error)
IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error
IDAllocDataReader(ctx context.Context) (io.ReadCloser, error)
IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error)
FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error)
StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
FinishTransaction(ctx context.Context, id string) (*Transaction, error)
Transactions(ctx context.Context) (map[string]*Transaction, error)
GetTransaction(ctx context.Context, id string) (*Transaction, error)
GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error)
GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error)
ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error
ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error
// SetInternalAPI tells the client the API it should use for internal/loopback ops
// where applicable.
SetInternalAPI(api *API)
}
//===============
// InternalQueryClient is the internal interface for querying a node.
type InternalQueryClient interface {
SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error)
QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error)
// Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist.
TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error)
TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, id []uint64) ([]string, error)
FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error)
CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error)
MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, like string) ([]uint64, error)
}
type nopInternalQueryClient struct{}
func (nopInternalQueryClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bool) ([]*IndexInfo, error) {
return nil, nil
}
func (n nopInternalQueryClient) QueryNode(ctx context.Context, uri *pnet.URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n nopInternalQueryClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, index, field string, keys []string, writable bool) ([]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, index, field string, ids []uint64) ([]string, error) {
return nil, nil
}
func (n nopInternalQueryClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, index string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalQueryClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, index string, field string, like string) ([]uint64, error) {
return nil, nil
}
func newNopInternalQueryClient() nopInternalQueryClient {
return nopInternalQueryClient{}
}
var _ InternalQueryClient = newNopInternalQueryClient()
//===============
type nopInternalClient struct{ nopInternalQueryClient }
func newNopInternalClient() nopInternalClient {
return nopInternalClient{}
}
var _ InternalClient = newNopInternalClient()
func (n nopInternalClient) AvailableShards(ctx context.Context, indexName string) ([]uint64, error) {
return nil, nil
}
func (n nopInternalClient) MaxShardByIndex(context.Context) (map[string]uint64, error) {
return nil, nil
}
func (n nopInternalClient) Schema(ctx context.Context) ([]*IndexInfo, error) { return nil, nil }
func (n nopInternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *Schema, remote bool) error {
return nil
}
func (n nopInternalClient) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
return nil
}
func (n nopInternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*topology.Node, error) {
return nil, nil
}
func (n nopInternalClient) PartitionNodes(ctx context.Context, partitionID int) ([]*topology.Node, error) {
return nil, nil
}
func (n nopInternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) {
return nil, nil
}
func (n nopInternalClient) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) {
return nil, nil
}
func (n nopInternalClient) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, options *ImportOptions) error {
return nil
}
func (n nopInternalClient) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, options *ImportOptions) error {
return nil
}
func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error {
return nil
}
func (n nopInternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, index, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) {
return nil, nil
}
func (n nopInternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error {
return nil
}
func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {
return nil
}
func (n nopInternalClient) EnsureField(ctx context.Context, indexName string, fieldName string) error {
return nil
}
func (n nopInternalClient) EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
return nil
}
func (n nopInternalClient) ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error {
return nil
}
func (n nopInternalClient) CreateField(ctx context.Context, index, field string) error { return nil }
func (n nopInternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error {
return nil
}
func (n nopInternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64) ([]FragmentBlock, error) {
return nil, nil
}
func (n nopInternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) {
return nil, nil, nil
}
func (n nopInternalClient) SendMessage(ctx context.Context, uri *pnet.URI, msg []byte) error {
return nil
}
func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pnet.URI) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri pnet.URI) (io.ReadCloser, error) {
return nil, nil
}
func (n nopInternalClient) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
return nil, nil
}
func (n nopInternalClient) FinishTransaction(ctx context.Context, id string) (*Transaction, error) {
return nil, nil
}
func (n nopInternalClient) Transactions(ctx context.Context) (map[string]*Transaction, error) {
return nil, nil
}
func (n nopInternalClient) GetTransaction(ctx context.Context, id string) (*Transaction, error) {
return nil, nil
}
func (n nopInternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) {
return nil, nil
}
func (n nopInternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) {
return nil, nil
}
func (c nopInternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, index, field string, remote bool, rddbdata io.Reader) error {
return nil
}
func (c nopInternalClient) ImportIndexKeys(ctx context.Context, uri *pnet.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error {
return nil
}
func (c nopInternalClient) SetInternalAPI(api *API) {
}

55
client/batch.go
View file

@ -2,12 +2,13 @@
package client
import (
"sort"
"sync"
"time"
"github.com/molecula/featurebase/v2/client/egpool"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v3/client/egpool"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/roaring"
"github.com/pkg/errors"
)
@ -20,7 +21,9 @@ const (
// order. Could be worth sorting everything after translation (as an
// option?). Instead of sorting all simultaneously, it might be faster
// (more cache friendly) to sort ids and save the swap ops to apply to
// everything else that needs to be sorted.
// everything else that needs to be sorted. Note: we're already doing
// some sorting in importValueData and importMutexData, so if we
// implement it at the top level, remember to remove it there.
// TODO support clearing values? nil values in records are ignored,
// but perhaps we could have a special type indicating that a bit or
@ -1216,6 +1219,22 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
return frags, clearFrags, nil
}
type valsByIDsSortable struct {
ids []uint64
vals []int64
// shard width so we can compare by shard instead of ID
width uint64
}
func (v *valsByIDsSortable) Len() int { return len(v.ids) }
// comparing on shard rather than ID was twice as fast in informal tests
func (v *valsByIDsSortable) Less(i, j int) bool { return v.ids[i]/v.width < v.ids[j]/v.width }
func (v *valsByIDsSortable) Swap(i, j int) {
v.ids[i], v.ids[j] = v.ids[j], v.ids[i]
v.vals[i], v.vals[j] = v.vals[j], v.vals[i]
}
// importValueData imports data for int fields.
func (b *Batch) importValueData() error {
shardWidth := b.index.ShardWidth()
@ -1246,6 +1265,12 @@ func (b *Batch) importValueData() error {
if len(ids) == 0 {
continue // TODO test this "all nil" case
}
sc := &valsByIDsSortable{ids: ids, vals: bvalues, width: shardWidth}
if !sort.IsSorted(sc) {
sort.Sort(sc)
}
curShard := ids[0] / shardWidth
startIdx := 0
for i := 1; i <= len(ids); i++ {
@ -1285,6 +1310,22 @@ func (b *Batch) importValueData() error {
return errors.Wrap(err, "importing value data")
}
type rowsByIDsSortable struct {
ids []uint64
rows []uint64
// shard width so we can compare by shard instead of ID
width uint64
}
func (v *rowsByIDsSortable) Len() int { return len(v.ids) }
// comparing on shard rather than ID was twice as fast in informal tests
func (v *rowsByIDsSortable) Less(i, j int) bool { return v.ids[i]/v.width < v.ids[j]/v.width }
func (v *rowsByIDsSortable) Swap(i, j int) {
v.ids[i], v.ids[j] = v.ids[j], v.ids[i]
v.rows[i], v.rows[j] = v.rows[j], v.rows[i]
}
// TODO this should work for bools as well - just need to support them
// at batch creation time and when calling Add, I think.
func (b *Batch) importMutexData() error {
@ -1319,6 +1360,12 @@ func (b *Batch) importMutexData() error {
if len(ids) == 0 {
continue
}
sc := &rowsByIDsSortable{ids: ids, rows: rowIDs, width: shardWidth}
if !sort.IsSorted(sc) {
sort.Sort(sc)
}
curShard := ids[0] / shardWidth
startIdx := 0
for i := 1; i <= len(ids); i++ {

163
client/batch_test.go
View file

@ -1,23 +1,48 @@
// Copyright 2021 Molecula Corp. All rights reserved.
//go:build integration
// +build integration
package client
import (
"math/rand"
"reflect"
"sort"
"strconv"
"testing"
"time"
"github.com/molecula/featurebase/v3/test"
"github.com/pkg/errors"
)
func TestStringSliceCombos(t *testing.T) {
client := DefaultClient()
func NewTestClient(t *testing.T, c *test.Cluster) *Client {
client, err := NewClient(c.Nodes[0].URL())
if err != nil {
t.Fatal(err)
}
return client
}
func TestAgainstCluster(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
client := NewTestClient(t, c)
t.Run("string-slice-combos", func(t *testing.T) { testStringSliceCombos(t, c, client) })
t.Run("import-batch-ints", func(t *testing.T) { testImportBatchInts(t, c, client) })
t.Run("import-batch-sorting", func(t *testing.T) { testImportBatchSorting(t, c, client) })
t.Run("test-trim-null", func(t *testing.T) { testTrimNull(t, c, client) })
t.Run("test-string-slice-empty-and-nil", func(t *testing.T) { testStringSliceEmptyAndNil(t, c, client) })
t.Run("test-string-slice", func(t *testing.T) { testStringSlice(t, c, client) })
t.Run("test-single-clear-batch-regression", func(t *testing.T) { testSingleClearBatchRegression(t, c, client) })
t.Run("test-batches", func(t *testing.T) { testBatches(t, c, client) })
t.Run("batches-strings-ids", func(t *testing.T) { testBatchesStringIDs(t, c, client) })
t.Run("test-batch-staleness", func(t *testing.T) { testBatchStaleness(t, c, client) })
}
func testStringSliceCombos(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("test-string-slicecombos")
idx := schema.Index("test-string-slice-combos")
fields := make([]*Field, 1)
fields[0] = idx.Field("a1", OptFieldKeys(true), OptFieldTypeSet(CacheTypeRanked, 100))
err := client.SyncSchema(schema)
@ -152,10 +177,9 @@ func ingestRecords(records []Row, batch *Batch) error {
return nil
}
func TestImportBatchInts(t *testing.T) {
client := DefaultClient()
func testImportBatchInts(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("gopilosatest-blah")
idx := schema.Index("test-import-batch-ints")
field := idx.Field("anint", OptFieldTypeInt())
err := client.SyncSchema(schema)
if err != nil {
@ -212,10 +236,59 @@ func TestImportBatchInts(t *testing.T) {
}
}
func TestTrimNull(t *testing.T) {
client := DefaultClient()
func testImportBatchSorting(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("gopilosatest-null")
idx := schema.Index("test-import-batch-sorting")
field := idx.Field("anint", OptFieldTypeInt())
field2 := idx.Field("amutex", OptFieldTypeMutex(CacheTypeNone, 0))
err := client.SyncSchema(schema)
if err != nil {
t.Fatalf("syncing schema: %v", err)
}
b, err := NewBatch(client, 100, idx, []*Field{field, field2})
if err != nil {
t.Fatalf("getting batch: %v", err)
}
r := Row{Values: make([]interface{}, 2)}
rnd := rand.New(rand.NewSource(7))
// generate 100 records randomly spread/ordered across multiple
// shards to test sorting on int/mutex fields
for i := 0; i < 100; i++ {
id := rnd.Intn(10_000_000)
r.ID = uint64(id)
r.Values[0] = int64(id)
r.Values[1] = uint64(id)
err := b.Add(r)
if err != nil && err != ErrBatchNowFull {
t.Fatalf("adding to batch: %v", err)
}
}
err = b.Import()
if err != nil {
t.Fatalf("importing: %v", err)
}
err = b.Import()
if err != nil {
t.Fatalf("second import: %v", err)
}
resp, err := client.Query(idx.RawQuery("Count(All())"))
if err != nil {
t.Fatalf("querying: %v", err)
}
if res := resp.Results()[0]; res.Count() != 100 {
t.Fatalf("unexpected result: %+v", res)
}
}
func testTrimNull(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("test-trim-null")
field := idx.Field("empty", OptFieldTypeInt())
err := client.SyncSchema(schema)
if err != nil {
@ -286,7 +359,7 @@ func TestTrimNull(t *testing.T) {
t.Fatalf("querying: %v", err)
}
for i, result := range resp.Results() {
if 1 == i {
if i == 1 {
if !reflect.DeepEqual(result.Row().Columns, []uint64(nil)) {
t.Errorf("expected %#v for %d, but got %#v", []uint64(nil), i, result.Row().Columns)
}
@ -299,8 +372,7 @@ func TestTrimNull(t *testing.T) {
}
func TestStringSliceEmptyAndNil(t *testing.T) {
client := DefaultClient()
func testStringSliceEmptyAndNil(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("test-string-slice-nil")
fields := make([]*Field, 1)
@ -397,8 +469,7 @@ func TestStringSliceEmptyAndNil(t *testing.T) {
}
func TestStringSlice(t *testing.T) {
client := DefaultClient()
func testStringSlice(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("test-string-slice")
fields := make([]*Field, 1)
@ -513,10 +584,9 @@ func TestStringSlice(t *testing.T) {
}
}
func TestSingleClearBatchRegression(t *testing.T) {
client := DefaultClient()
func testSingleClearBatchRegression(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("gopilosatest-blah")
idx := schema.Index("test-single-clear-batch-regression")
numFields := 1
fields := make([]*Field, numFields)
fields[0] = idx.Field("zero", OptFieldKeys(true))
@ -565,10 +635,9 @@ func TestSingleClearBatchRegression(t *testing.T) {
}
func TestBatches(t *testing.T) {
client := DefaultClient()
func testBatches(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("gopilosatest-blah")
idx := schema.Index("test-batches")
numFields := 5
fields := make([]*Field, numFields)
fields[0] = idx.Field("zero", OptFieldKeys(true))
@ -890,8 +959,8 @@ func TestBatches(t *testing.T) {
}
}
res := results[1]
cols := res.Row().Columns
if !reflect.DeepEqual(cols, []uint64{0, 2, 4, 6, 8, 10, 12, 14, 16, 18}) {
if cols := res.Row().Columns; !reflect.DeepEqual(cols, []uint64{0, 2, 4, 6, 8, 10, 12, 14, 16, 18}) {
t.Fatalf("unexpected columns for field 1 row b: %v", cols)
}
@ -919,23 +988,25 @@ func TestBatches(t *testing.T) {
t.Fatalf("querying: %v", err)
}
results = resp.Results()
cols = results[0].Row().Columns
if !reflect.DeepEqual(cols, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}) {
if cols := results[0].Row().Columns; !reflect.DeepEqual(cols, []uint64{0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}) {
t.Fatalf("all columns (but 8) should be greater than -11, but got: %v", cols)
}
cols = results[1].Row().Columns
if !reflect.DeepEqual(cols, []uint64{19, 21, 23, 25, 27}) {
if cols := results[1].Row().Columns; !reflect.DeepEqual(cols, []uint64{19, 21, 23, 25, 27}) {
t.Fatalf("wrong cols for ==0: %v", cols)
}
cols = results[2].Row().Columns
if !reflect.DeepEqual(cols, []uint64{20, 22, 24, 26, 28}) {
if cols := results[2].Row().Columns; !reflect.DeepEqual(cols, []uint64{20, 22, 24, 26, 28}) {
t.Fatalf("wrong cols for ==100: %v", cols)
}
cols = results[3].Row().Columns
cols := results[3].Row().Columns
exp := []uint64{0, 2, 4, 6, 10, 12, 14, 16, 18}
if !reflect.DeepEqual(cols, exp) {
t.Fatalf("wrong cols for January: got/want\n%v\n%v", cols, exp)
}
cols = results[4].Row().Columns
exp = []uint64{1, 3, 5, 7}
if !reflect.DeepEqual(cols, exp) {
@ -977,10 +1048,9 @@ func TestBatches(t *testing.T) {
// TODO test importing across multiple shards
}
func TestBatchesStringIDs(t *testing.T) {
client := DefaultClient()
func testBatchesStringIDs(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("gopilosatest-blah", OptIndexKeys(true))
idx := schema.Index("batches-strings-ids", OptIndexKeys(true))
fields := make([]*Field, 3)
fields[0] = idx.Field("zero", OptFieldKeys(true))
fields[1] = idx.Field("one", OptFieldTypeMutex(CacheTypeNone, 0), OptFieldKeys(true))
@ -1117,26 +1187,6 @@ outer:
return nil
}
func isPermutationOfInt(one, two []uint64) error {
if len(one) != len(two) {
return errors.Errorf("different lengths %d and %d", len(one), len(two))
}
outer:
for _, vOne := range one {
for j, vTwo := range two {
if vOne == vTwo {
two = append(two[:j], two[j+1:]...)
continue outer
}
}
return errors.Errorf("%d in one but not two", vOne)
}
if len(two) != 0 {
return errors.Errorf("vals in two but not one: %v", two)
}
return nil
}
func TestQuantizedTime(t *testing.T) {
cases := []struct {
name string
@ -1263,10 +1313,9 @@ func TestQuantizedTime(t *testing.T) {
}
func TestBatchStaleness(t *testing.T) {
client := DefaultClient()
func testBatchStaleness(t *testing.T, c *test.Cluster, client *Client) {
schema := NewSchema()
idx := schema.Index("gopilosatest-blah")
idx := schema.Index("test-batch-staleness")
field := idx.Field("anint", OptFieldTypeInt())
err := client.SyncSchema(schema)
if err != nil {

50
client/client.go
View file

@ -20,13 +20,13 @@ import (
"time"
"github.com/golang/protobuf/proto" //nolint:staticcheck
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/logger"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/pb"
"github.com/molecula/featurebase/v2/pql"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/stats"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/logger"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/pb"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -36,7 +36,7 @@ import (
const PQLVersion = "1.0"
// DefaultShardWidth is used if an index doesn't have it defined.
const DefaultShardWidth = 1 << 20
const DefaultShardWidth = pilosa.ShardWidth
const maxHosts = 10
@ -61,6 +61,8 @@ type Client struct {
shardNodes shardNodes
tick *time.Ticker
done chan struct{}
AuthToken string
}
func (c *Client) getURIsForShard(index string, shard uint64) ([]*pnet.URI, error) {
@ -283,7 +285,7 @@ func (c *Client) Query(query PQLQuery, options ...interface{}) (*QueryResponse,
return nil, errors.Wrap(err, "making request data")
}
path := fmt.Sprintf("/index/%s/query", query.Index().name)
_, respData, err := c.HTTPRequest("POST", path, reqData, defaultProtobufHeaders())
_, respData, err := c.HTTPRequest("POST", path, reqData, c.augmentHeaders(defaultProtobufHeaders()))
if err != nil {
return nil, err
}
@ -306,7 +308,7 @@ func (c *Client) CreateIndex(index *Index) error {
data := []byte(index.options.String())
path := fmt.Sprintf("/index/%s", index.name)
status, body, err := c.HTTPRequest("POST", path, data, nil)
status, body, err := c.HTTPRequest("POST", path, data, c.augmentHeaders(nil))
if err != nil {
return errors.Wrapf(err, "creating index: %s", index.name)
}
@ -330,7 +332,7 @@ func (c *Client) CreateField(field *Field) error {
data := []byte(field.options.String())
path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name)
status, body, err := c.HTTPRequest("POST", path, data, nil)
status, body, err := c.HTTPRequest("POST", path, data, c.augmentHeaders(nil))
if err != nil {
return errors.Wrapf(err, "creating field: %s in index: %s", field.name, field.index.name)
}
@ -398,7 +400,7 @@ func (c *Client) DeleteIndexByName(index string) error {
defer span.Finish()
path := fmt.Sprintf("/index/%s", index)
_, _, err := c.HTTPRequest("DELETE", path, nil, nil)
_, _, err := c.HTTPRequest("DELETE", path, nil, c.augmentHeaders(nil))
return err
}
@ -408,7 +410,7 @@ func (c *Client) DeleteField(field *Field) error {
defer span.Finish()
path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name)
_, _, err := c.HTTPRequest("DELETE", path, nil, nil)
_, _, err := c.HTTPRequest("DELETE", path, nil, c.augmentHeaders(nil))
return err
}
@ -597,7 +599,7 @@ func (c *Client) fetchFragmentNodes(indexName string, shard uint64) ([]fragmentN
return []fragmentNode{*c.manualFragmentNode}, nil
}
path := fmt.Sprintf("/internal/fragment/nodes?shard=%d&index=%s", shard, indexName)
_, body, err := c.HTTPRequest("GET", path, []byte{}, nil)
_, body, err := c.HTTPRequest("GET", path, []byte{}, c.augmentHeaders(nil))
if err != nil {
return nil, err
}
@ -635,7 +637,7 @@ func (c *Client) fetchPrimaryNode() (fragmentNode, error) {
}
func (c *Client) importData(uri *pnet.URI, path string, data []byte) error {
if status, _, err := c.doRequest(uri, "POST", path, defaultProtobufHeaders(), data); err != nil {
if status, _, err := c.doRequest(uri, "POST", path, c.augmentHeaders(defaultProtobufHeaders()), data); err != nil {
return errors.Wrapf(err, "import to %s", uri.HostPort())
} else if status == http.StatusPreconditionFailed {
return ErrPreconditionFailed
@ -683,7 +685,8 @@ func (c *Client) importRoaringBitmap(uri *pnet.URI, field *Field, shard uint64,
return err
}
status, _, err := c.doRequest(uri, "POST", path, defaultProtobufHeaders(), data)
header := c.augmentHeaders(defaultProtobufHeaders())
status, _, err := c.doRequest(uri, "POST", path, header, data)
if err != nil {
return errors.Wrapf(err, "roaring import to %s, status: %d", uri.HostPort(), status)
}
@ -724,7 +727,7 @@ func (c *Client) Info() (Info, error) {
span := c.tracer.StartSpan("Client.Info")
defer span.Finish()
_, data, err := c.HTTPRequest("GET", "/info", nil, nil)
_, data, err := c.HTTPRequest("GET", "/info", nil, c.augmentHeaders(nil))
if err != nil {
return Info{}, errors.Wrap(err, "requesting /info")
}
@ -754,7 +757,7 @@ func (c *Client) Status() (Status, error) {
}
func (c *Client) readSchema() ([]SchemaIndex, error) {
_, data, err := c.HTTPRequest("GET", "/schema", nil, nil)
_, data, err := c.HTTPRequest("GET", "/schema", nil, c.augmentHeaders(nil))
if err != nil {
return nil, errors.Wrap(err, "requesting /schema")
}
@ -1021,6 +1024,9 @@ func (c *Client) augmentHeaders(headers map[string]string) map[string]string {
version := strings.TrimPrefix(Version, "v")
headers["User-Agent"] = fmt.Sprintf("pilosa/client/%s", version)
if c.AuthToken != "" {
headers["Authorization"] = c.AuthToken
}
return headers
}
@ -1177,7 +1183,7 @@ func (c *Client) startTransaction(id string, timeout time.Duration, exclusive bo
return nil, errors.Wrap(err, "marshalling transaction")
}
status, data, err := c.httpRequest("POST", "/transaction", bod, defaultJSONHeaders(), true)
status, data, err := c.httpRequest("POST", "/transaction", bod, c.augmentHeaders(defaultJSONHeaders()), true)
if status == http.StatusConflict && time.Now().Before(deadline) {
// if we're getting StatusConflict after all the usual timeouts/retries, keep retrying until the deadline
time.Sleep(time.Second)
@ -1204,7 +1210,7 @@ func (c *Client) startTransaction(id string, timeout time.Duration, exclusive bo
}
func (c *Client) FinishTransaction(id string) (*pilosa.Transaction, error) {
_, data, err := c.httpRequest("POST", "/transaction/"+id+"/finish", nil, defaultJSONHeaders(), true)
_, data, err := c.httpRequest("POST", "/transaction/"+id+"/finish", nil, c.augmentHeaders(defaultJSONHeaders()), true)
if err != nil && len(data) == 0 {
return nil, err
}
@ -1226,7 +1232,7 @@ func (c *Client) FinishTransaction(id string) (*pilosa.Transaction, error) {
}
func (c *Client) Transactions() (map[string]*pilosa.Transaction, error) {
_, respData, err := c.httpRequest("GET", "/transactions", nil, defaultJSONHeaders(), true)
_, respData, err := c.httpRequest("GET", "/transactions", nil, c.augmentHeaders(defaultJSONHeaders()), true)
if err != nil {
return nil, errors.Wrap(err, "getting transactions")
}
@ -1240,7 +1246,7 @@ func (c *Client) Transactions() (map[string]*pilosa.Transaction, error) {
}
func (c *Client) GetTransaction(id string) (*pilosa.Transaction, error) {
_, data, err := c.httpRequest("GET", "/transaction/"+id, nil, defaultJSONHeaders(), true)
_, data, err := c.httpRequest("GET", "/transaction/"+id, nil, c.augmentHeaders(defaultJSONHeaders()), true)
if err != nil {
return nil, err
}

8
client/client_it_test.go
View file

@ -7,10 +7,10 @@ import (
"testing"
"time"
"github.com/molecula/featurebase/v2/disco"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/shardwidth"
"github.com/molecula/featurebase/v2/test"
"github.com/molecula/featurebase/v3/disco"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/shardwidth"
"github.com/molecula/featurebase/v3/test"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)

2
client/client_test.go
View file

@ -10,7 +10,7 @@ import (
"reflect"
"testing"
pnet "github.com/molecula/featurebase/v2/net"
pnet "github.com/molecula/featurebase/v3/net"
)
func TestQueryWithError(t *testing.T) {

2
client/cluster.go
View file

@ -7,7 +7,7 @@ package client
import (
"sync"
pnet "github.com/molecula/featurebase/v2/net"
pnet "github.com/molecula/featurebase/v3/net"
)
// Cluster contains hosts in a Pilosa cluster.

2
client/cluster_test.go
View file

@ -7,7 +7,7 @@ package client
import (
"testing"
pnet "github.com/molecula/featurebase/v2/net"
pnet "github.com/molecula/featurebase/v3/net"
)
func TestNewClusterWithHost(t *testing.T) {

2
client/csv/csv.go
View file

@ -10,7 +10,7 @@ import (
"strings"
"time"
"github.com/molecula/featurebase/v2/client"
"github.com/molecula/featurebase/v3/client"
)
// Format is the format of the data in the CSV file.

4
client/csv/csv_it_test.go
View file

@ -10,8 +10,8 @@ import (
"strings"
"testing"
"github.com/molecula/featurebase/v2/client"
"github.com/molecula/featurebase/v2/client/csv"
"github.com/molecula/featurebase/v3/client"
"github.com/molecula/featurebase/v3/client/csv"
)
func TestCSVIterate(t *testing.T) {

6
client/csv/csv_test.go
View file

@ -8,9 +8,9 @@ import (
"strings"
"testing"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/client"
"github.com/molecula/featurebase/v2/client/csv"
"github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/client"
"github.com/molecula/featurebase/v3/client/csv"
)
func TestCSVColumnIterator(t *testing.T) {

2
client/doc.go
View file

@ -11,7 +11,7 @@ Usage:
import (
"fmt"
"github.com/molecula/featurebase/v2/client"
"github.com/molecula/featurebase/v3/client"
)
// Create a Client instance

2
client/egpool/egpool_test.go
View file

@ -5,7 +5,7 @@ import (
"errors"
"testing"
"github.com/molecula/featurebase/v2/client/egpool"
"github.com/molecula/featurebase/v3/client/egpool"
)
func TestEGPool(t *testing.T) {

2
client/ingest_api_batch.go
View file

@ -3,7 +3,7 @@ package client
import (
"time"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v3/logger"
"github.com/pkg/errors"
)

17
client/ingest_api_batch_test.go
View file

@ -5,8 +5,8 @@ import (
"testing"
"time"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/test"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/test"
)
func TestIngestAPIBatchAdd(t *testing.T) {
@ -133,6 +133,7 @@ func TestIngestAPIBatchAdd(t *testing.T) {
}
func TestIngestAPIBatch(t *testing.T) {
t.Skip("causing sporadic CI failures... on my list to debug, but this code doesn't affect anyone's production anyhow (jaffee)")
c := test.MustRunCluster(t, 3)
defer c.Close()
@ -269,37 +270,37 @@ func TestIngestAPIBatch(t *testing.T) {
if resp, err := cli.Query(NewPQLBaseQuery("Row(bint==-2)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
t.Fatalf("unexpected Row(bint==-2) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(cid=9)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
t.Fatalf("unexpected Row(cid=9) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(dtimestamp=='2010-10-18T02:07:03Z')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
t.Fatalf("unexpected Row(dtimestamp=='2010-10-18T02:07:03Z') result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(etime=e, from='2010-01-01', to='2010-01-02')", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
t.Fatalf("unexpected Row(etime=e, from='2010-01-01', to='2010-01-02') result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(fdecimal==1.234)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
t.Fatalf("unexpected Row(fdecimal==1.234) result: %+v", resp.Result().Row().Columns)
}
if resp, err := cli.Query(NewPQLBaseQuery("Row(gbool=true)", &Index{name: "test-1", options: &IndexOptions{}}, nil)); err != nil {
t.Fatalf("querying: %v", err)
} else if len(resp.Result().Row().Columns) != 1 || resp.Result().Row().Columns[0] != uint64(7) {
t.Fatalf("unexpected Row(asr=a) result: %+v", resp.Result().Row().Columns)
t.Fatalf("unexpected Row(gbool=true) result: %+v", resp.Result().Row().Columns)
}
}

2
client/orm.go
View file

@ -13,7 +13,7 @@ import (
"sync"
"time"
"github.com/molecula/featurebase/v2/pql"
"github.com/molecula/featurebase/v3/pql"
"github.com/pkg/errors"
)

4
client/orm_test.go
View file

@ -13,8 +13,8 @@ import (
"testing"
"time"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/pql"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/pql"
"github.com/pkg/errors"
)

2
client/record_test.go
View file

@ -7,7 +7,7 @@ package client_test
import (
"testing"
"github.com/molecula/featurebase/v2/client"
"github.com/molecula/featurebase/v3/client"
)
func TestColumnShard(t *testing.T) {

2
client/response.go
View file

@ -8,7 +8,7 @@ import (
"encoding/json"
"fmt"
"github.com/molecula/featurebase/v2/pb"
"github.com/molecula/featurebase/v3/pb"
)
// QueryResponse types.

2
client/response_test.go
View file

@ -11,7 +11,7 @@ import (
"reflect"
"testing"
"github.com/molecula/featurebase/v2/pb"
"github.com/molecula/featurebase/v3/pb"
)
func TestNewRowResultFromInternal(t *testing.T) {

2
client/shardnodes.go
View file

@ -7,7 +7,7 @@ package client
import (
"sync"
pnet "github.com/molecula/featurebase/v2/net"
pnet "github.com/molecula/featurebase/v3/net"
)
type shardNodes struct {

16
cluster.go
View file

@ -10,12 +10,12 @@ import (
"sync"
"time"
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/topology"
"github.com/molecula/featurebase/v2/tracing"
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/ingest"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/topology"
"github.com/molecula/featurebase/v3/tracing"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -102,7 +102,7 @@ type cluster struct { // nolint: maligned
logger logger.Logger
InternalClient InternalClient
InternalClient *InternalClient
confirmDownRetries int
confirmDownSleep time.Duration
@ -120,7 +120,7 @@ func newCluster() *cluster {
translationSyncer: NopTranslationSyncer,
InternalClient: newNopInternalClient(),
InternalClient: &InternalClient{}, // TODO might have to fill this out a bit
logger: logger.NopLogger,

10
cluster_internal_test.go
View file

@ -12,11 +12,11 @@ import (
"time"
"github.com/davecgh/go-spew/spew"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/testhook"
"github.com/molecula/featurebase/v2/topology"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/testhook"
"github.com/molecula/featurebase/v3/topology"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
)
// Ensure that fragCombos creates the correct fragment mapping.

2
cmd.go
View file

@ -4,7 +4,7 @@ package pilosa
import (
"io"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v3/logger"
)
// CmdIO holds standard unix inputs and outputs.

15
cmd/backup.go
View file

@ -5,7 +5,7 @@ import (
"context"
"io"
"github.com/molecula/featurebase/v2/ctl"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
@ -23,11 +23,14 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
}
flags := ccmd.Flags()
flags.StringVarP(&cmd.OutputDir, "output", "o", "", "output dir to write to")
flags.BoolVar(&cmd.NoSync, "no-sync", false, "disable file sync")
flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "number of concurrent backup goroutines")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
flags.StringVar(&cmd.Index, "index", "", "index to backup, default backs up all indexes. ")
flags.StringVarP(&cmd.OutputDir, "output", "o", "", "Output directory to write to.")
flags.BoolVar(&cmd.NoSync, "no-sync", false, "Disable file sync")
flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "Number of concurrent backup goroutines.")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "The address (host:port) of FeatureBase (HTTP).")
flags.StringVar(&cmd.Index, "index", "", "Index to backup, default backs up all indexes. ")
flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
return ccmd
}

12
cmd/badloader/badloader.go
View file

@ -12,17 +12,17 @@ import (
"io/ioutil"
gohttp "net/http"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/http"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/vprint"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/encoding/proto"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/vprint"
"os"
"strconv"
"strings"
)
func UploadTar(srcFile string, client *http.InternalClient) error {
func UploadTar(srcFile string, client *pilosa.InternalClient) error {
t0 := time.Now()
f, err := os.Open(srcFile)
if err != nil {
@ -114,7 +114,7 @@ func main() {
host := "127.0.0.1:10101"
h := &gohttp.Client{}
c, err := http.NewInternalClient(host, h)
c, err := pilosa.NewInternalClient(host, h, pilosa.WithSerializer(proto.Serializer{}))
vprint.PanicOn(err)
tarSrcPath := "q2.tar.gz"

33
cmd/check.go
View file

@ -1,33 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (
"context"
"fmt"
"io"
"github.com/spf13/cobra"
"github.com/molecula/featurebase/v2/ctl"
)
var checker *ctl.CheckCommand
func newCheckCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
checker = ctl.NewCheckCommand(stdin, stdout, stderr)
checkCmd := &cobra.Command{
Use: "check <path> [path2]...",
Short: "Do a consistency check on a FeatureBase data file.",
Long: `
Performs a consistency check on data files.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("path required")
}
checker.Paths = args
return checker.Run(context.Background())
},
}
return checkCmd
}

23
cmd/check_test.go
View file

@ -1,23 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (
"strings"
"testing"
)
func TestCheckHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "check", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "featurebase check") || err != nil {
t.Fatalf("Command 'check --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestCheckNoPath(t *testing.T) {
output, err := ExecNewRootCommand(t, "check")
if !strings.Contains(err.Error(), "path required") {
t.Fatalf("Command 'check' without args should error but: err: '%v', output: '%v'", err, output)
}
}

2
cmd/chksum.go
View file

@ -5,7 +5,7 @@ import (
"context"
"io"
"github.com/molecula/featurebase/v2/ctl"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)

4
cmd/config.go
View file

@ -7,8 +7,8 @@ import (
"github.com/spf13/cobra"
"github.com/molecula/featurebase/v2/ctl"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v3/ctl"
"github.com/molecula/featurebase/v3/server"
)
var conf *ctl.ConfigCommand

43
cmd/convert.go
View file

@ -1,43 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (
"context"
"fmt"
"io"
"github.com/spf13/cobra"
"github.com/molecula/featurebase/v2/ctl"
)
var inspector *ctl.InspectCommand
func newInspectCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
inspector = ctl.NewInspectCommand(stdin, stdout, stderr)
inspectCmd := &cobra.Command{
Use: "inspect",
Short: "Get stats on a FeatureBase data file.",
Long: `
Inspects a data file and provides stats.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return fmt.Errorf("path required")
} else if len(args) > 1 {
return fmt.Errorf("only one path allowed")
}
inspector.Path = args[0]
return inspector.Run(context.Background())
},
}
flags := inspectCmd.Flags()
flags.BoolVarP(&inspector.Quiet, "quiet", "q", false, "don't list details of containers")
flags.IntVarP(&inspector.Max, "max", "n", 0, "list at most max items (0 = unlimited)")
flags.StringVarP(&inspector.InspectOpts.Indexes, "index", "i", "", "filter indexes")
flags.StringVarP(&inspector.InspectOpts.Views, "view", "v", "", "filter views")
flags.StringVarP(&inspector.InspectOpts.Fields, "field", "f", "", "filter fields")
flags.StringVarP(&inspector.InspectOpts.Shards, "shard", "s", "", "filter shards")
return inspectCmd
}

2
cmd/export.go
View file

@ -7,7 +7,7 @@ import (
"github.com/spf13/cobra"
"github.com/molecula/featurebase/v2/ctl"
"github.com/molecula/featurebase/v3/ctl"
)
var Exporter *ctl.ExportCommand

2
cmd/export_test.go
View file

@ -5,7 +5,7 @@ import (
"strings"
"testing"
"github.com/molecula/featurebase/v2/cmd"
"github.com/molecula/featurebase/v3/cmd"
)
func TestExportHelp(t *testing.T) {

2
cmd/featurebase-parse-sql/main.go
View file

@ -9,7 +9,7 @@ import (
"os"
"strings"
"github.com/molecula/featurebase/v2/sql2"
"github.com/molecula/featurebase/v3/sql2"
)
func main() {

2
cmd/featurebase/main.go
View file

@ -8,7 +8,7 @@ import (
"fmt"
"os"
"github.com/molecula/featurebase/v2/cmd"
"github.com/molecula/featurebase/v3/cmd"
)
func main() {

13
cmd/featurebase/main_test.go Normal file
View file

@ -0,0 +1,13 @@
//go:build testrunmain
// +build testrunmain
package main
import (
"testing"
)
// Wrapper test for main function used to get code coverage for end2end tests
func TestRunMain(t *testing.T) {
main()
}

2
cmd/generate_config.go
View file

@ -7,7 +7,7 @@ import (
"github.com/spf13/cobra"
"github.com/molecula/featurebase/v2/ctl"
"github.com/molecula/featurebase/v3/ctl"
)
var generateConf *ctl.GenerateConfigCommand

5
cmd/import.go
View file

@ -5,8 +5,8 @@ import (
"context"
"io"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/ctl"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
@ -51,6 +51,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
ctl.SetTLSConfig(flags, "", &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
flags.StringVar(&Importer.AuthToken, "auth-token", "", "Authentication token")
return importCmd
}

6
cmd/import_test.go
View file

@ -5,10 +5,10 @@ import (
"strings"
"testing"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v2/cmd"
"github.com/molecula/featurebase/v2/pql"
"github.com/molecula/featurebase/v3/cmd"
"github.com/molecula/featurebase/v3/pql"
)
func TestImportHelp(t *testing.T) {

29
cmd/inspect_test.go
View file

@ -1,29 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd_test
import (
"strings"
"testing"
)
func TestInspectHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "inspect", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "featurebase inspect") || err != nil {
t.Fatalf("Command 'inspect --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestInspectNoPath(t *testing.T) {
output, err := ExecNewRootCommand(t, "inspect")
if !strings.Contains(err.Error(), "path required") {
t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output)
}
}
func TestInspectMultiPath(t *testing.T) {
output, err := ExecNewRootCommand(t, "inspect", "one", "two")
if !strings.Contains(err.Error(), "only one path") {
t.Fatalf("Command 'inspect' without args should error but: err: '%v', output: '%v'", err, output)
}
}

28
cmd/keygen.go Normal file
View file

@ -0,0 +1,28 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (
"context"
"io"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
func newKeygenCommand(stdin io.Reader, stdout io.Writer, stderr io.Writer) *cobra.Command {
cmd := ctl.NewKeygenCommand(stdin, stdout, stderr)
ccmd := &cobra.Command{
Use: "keygen",
Short: "Generate secret key for authentication.",
Long: `
Generate secret key to configure FeatureBase for Authentication.
`,
RunE: func(c *cobra.Command, args []string) error {
return cmd.Run(context.Background())
},
}
flags := ccmd.Flags()
flags.IntVarP(&cmd.KeyLength, "length", "l", 32, "length of the key to produce")
return ccmd
}

10
cmd/pilosa-bench/main.go
View file

@ -16,8 +16,8 @@ import (
"strings"
"time"
"github.com/molecula/featurebase/v2"
phttp "github.com/molecula/featurebase/v2/http"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/encoding/proto"
"golang.org/x/sync/errgroup"
)
@ -78,7 +78,7 @@ func run(ctx context.Context, args []string) (err error) {
rand.Seed(0)
// Setup connection to pilosa.
client, err := phttp.NewInternalClient(*hostport, http.DefaultClient)
client, err := pilosa.NewInternalClient(*hostport, http.DefaultClient, pilosa.WithSerializer(proto.Serializer{}))
if err != nil {
return err
}
@ -270,7 +270,7 @@ func generateTopKQuery(index, field string, from, to time.Time) string {
}
// loadFields returns a mapping of index/field names to field info & identifiers.
func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey]*fieldInfo, error) {
func loadFields(ctx context.Context, client *pilosa.InternalClient) (map[fieldKey]*fieldInfo, error) {
indexes, err := client.Schema(ctx)
if err != nil {
return nil, err
@ -299,7 +299,7 @@ func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey
}
// fetchFieldIDs returns a list of field IDs or keys.
func fetchFieldIDs(ctx context.Context, client *phttp.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
func fetchFieldIDs(ctx context.Context, client *pilosa.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) {
resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`})
if err != nil {
return nil, err

15
cmd/random-query/main.go
View file

@ -16,12 +16,13 @@ import (
"time"
"github.com/gogo/protobuf/proto"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/client"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/pb"
"github.com/molecula/featurebase/v2/pql"
"github.com/molecula/featurebase/v2/vprint"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/client"
fb_proto "github.com/molecula/featurebase/v3/encoding/proto"
"github.com/molecula/featurebase/v3/pb"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
vegeta "github.com/tsenart/vegeta/v12/lib"
)
@ -162,7 +163,7 @@ func main() {
func (cfg *RandomQueryConfig) Run() (err error) {
remoteClient := nethttp.DefaultClient
cli, err := http.NewInternalClient(cfg.HostPort, remoteClient)
cli, err := pilosa.NewInternalClient(cfg.HostPort, remoteClient, pilosa.WithSerializer(fb_proto.Serializer{}))
if err != nil {
return err
}

13
cmd/random-query/main_test.go
View file

@ -7,12 +7,11 @@ import (
"strconv"
"testing"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/boltdb"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/test"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/boltdb"
"github.com/molecula/featurebase/v3/server"
"github.com/molecula/featurebase/v3/test"
. "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck
)
func Test_RandomQuery(t *testing.T) {
@ -35,7 +34,7 @@ func Test_RandomQuery(t *testing.T) {
server.OptCommandServerOptions(
pilosa.OptServerNodeID(nodeid[0]),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerOpenTranslateReader(pilosa.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(nReplicas),
)},
)

2
cmd/rbf.go
View file

@ -8,7 +8,7 @@ import (
"io"
"strconv"
"github.com/molecula/featurebase/v2/ctl"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)

5
cmd/restore.go
View file

@ -5,7 +5,7 @@ import (
"context"
"io"
"github.com/molecula/featurebase/v2/ctl"
"github.com/molecula/featurebase/v3/ctl"
"github.com/spf13/cobra"
)
@ -25,6 +25,9 @@ The Restore command will take a backup archive and restore it to a new, clean cl
flags.StringVarP(&cmd.Path, "source", "s", "", "backup file; specify '-' to restore from stdin tar stream")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
flags.IntVar(&cmd.Concurrency, "concurrency", 1, "number of concurrent uploads")
flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
ctl.SetTLSConfig(
flags, "",
&cmd.TLS.CertificatePath,

185
cmd/roaring-migrate/main.go
View file

@ -4,6 +4,7 @@ package main
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@ -12,26 +13,43 @@ import (
"strings"
"syscall"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/rbf"
"github.com/molecula/featurebase/v2/rbf/cfg"
"github.com/molecula/featurebase/v2/roaring"
txkey "github.com/molecula/featurebase/v2/short_txkey"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/rbf"
"github.com/molecula/featurebase/v3/rbf/cfg"
"github.com/molecula/featurebase/v3/roaring"
txkey "github.com/molecula/featurebase/v3/short_txkey"
"github.com/molecula/featurebase/v3/vprint"
"github.com/spf13/cobra"
)
var visited map[string]int64
var glogger = logger.NewStandardLogger(os.Stdout)
const (
Version = "1.0"
)
func main() {
os.Exit(realMain())
}
func realMain() int {
visited = make(map[string]int64)
var dataDir, backupPath string
var verbose bool
cmdMigrate := &cobra.Command{
Use: "roaring-migrate",
Short: "convert roaring pilosa backup to rbf",
Long: `roaring-migrate uses the pilosa data-dir for each node, and produces a new backup that is able to be restored from utilizing the new pilosa restore tool.`,
Run: func(cmd *cobra.Command, args []string) {
if verbose {
glogger.Infof("Version: %v", Version)
}
nodes := strings.Split(dataDir, ",")
for _, nodePath := range nodes {
err := Migrate(nodePath, backupPath)
err := Migrate(nodePath, backupPath, verbose)
if err != nil {
fmt.Println("Error", err)
glogger.Errorf("%v", Version)
return
}
@ -40,24 +58,24 @@ func main() {
}
cmdMigrate.Flags().StringVarP(&dataDir, "data-dir", "d", "", "source directories for each node seperated by commas")
cmdMigrate.Flags().StringVarP(&backupPath, "backup-dir", "b", "", "location of backup directory")
cmdMigrate.Flags().BoolVar(&verbose, "verbose", false, "additional progress information")
err := cmdMigrate.MarkFlagRequired("data-dir")
if err != nil {
fmt.Println("Error setting flag data-dir")
os.Exit(1)
return
glogger.Errorf("Error setting flag data-dir")
return 1
}
err = cmdMigrate.MarkFlagRequired("backup-dir")
if err != nil {
fmt.Println("Error setting flag backup-dir")
os.Exit(1)
return
glogger.Errorf("Error setting flag backup-dir")
return 1
}
err = cmdMigrate.Execute()
if err != nil {
fmt.Println("exec error", err)
os.Exit(1)
glogger.Errorf("exec error %v", err)
return 1
}
return 0
}
func FetchFragments(base string) []string {
@ -67,7 +85,7 @@ func FetchFragments(base string) []string {
// first thing to do, check error. and decide what to do about it
if errX != nil {
fmt.Printf("error 「%v」 at a path 「%q」\n", errX, pathX)
glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX)
return errX
}
pathX = pathX[len(base):]
@ -82,7 +100,7 @@ func FetchFragments(base string) []string {
err := filepath.Walk(base, ff)
if err != nil {
fmt.Printf("error walking the path %q: %v\n", base, err)
glogger.Errorf("error walking the path %q: %v\n", base, err)
}
return fragments
}
@ -94,6 +112,14 @@ type local struct {
Fields []*pilosa.FieldInfo `json:"fields,omitempty"`
}
func fileExists(filename string) (bool, int64) {
info, err := os.Stat(filename)
if os.IsNotExist(err) {
return false, 0
}
return !info.IsDir(), info.Size()
}
func BuildSchema(dataDir string) ([]byte, error) {
//need to find all the ".meta" files and load as field options
@ -105,7 +131,7 @@ func BuildSchema(dataDir string) ([]byte, error) {
// first thing to do, check error. and decide what to do about it
if errX != nil {
fmt.Printf("error 「%v」 at a path 「%q」\n", errX, pathX)
glogger.Infof("error 「%v」 at a path 「%q」\n", errX, pathX)
return errX
}
pathX = pathX[len(dataDir):]
@ -115,7 +141,7 @@ func BuildSchema(dataDir string) ([]byte, error) {
if strings.Contains(pathX, ".meta") {
//convert the file to a fieldOptions
// ex: metaPath /trait_store/aba/.meta
fmt.Println("PATHX", pathX)
glogger.Infof("PATHX %v", pathX)
t := strings.Split(pathX, "/")
index := t[1]
src := dataDir + pathX
@ -160,7 +186,7 @@ func BuildSchema(dataDir string) ([]byte, error) {
err := filepath.Walk(dataDir, ff)
if err != nil {
fmt.Printf("error walking the path %q: %v\n", dataDir, err)
glogger.Errorf("error walking the path %q: %v\n", dataDir, err)
}
return json.MarshalIndent(schemaSerializer, "", " ")
}
@ -183,7 +209,7 @@ func (d *rbfFile) getDB(path, index string, shard uint64) (*rbf.DB, error) {
if d.last != src {
d.Close()
d.last = src
fmt.Println("RBF:", src)
glogger.Infof("RBF: %v", src)
c := cfg.NewDefaultConfig()
c.FsyncEnabled = false
c.MinWALCheckpointSize = 0
@ -197,20 +223,34 @@ func (d *rbfFile) getDB(path, index string, shard uint64) (*rbf.DB, error) {
return d.working, nil
}
func (d *rbfFile) Close() error {
defer func() error {
// clean up the temp directory
err := os.RemoveAll(d.temp)
if err != nil {
return err
}
return nil
}()
if d.last != "" {
d.working.Close()
//if d.last exists only keep the biggest
err := os.MkdirAll(filepath.Dir(d.last), 0777)
if err != nil {
return err
exists, sz := fileExists(d.last)
src := filepath.Join(d.temp, "data")
if !exists {
err := os.MkdirAll(filepath.Dir(d.last), 0777)
if err != nil {
return err
}
} else {
_, sz2 := fileExists(src)
if sz > sz2 {
return nil
}
}
// move the datafile backup shard
err = os.Rename(filepath.Join(d.temp, "data"), d.last)
if err != nil {
return err
}
//cleanup the tempdirectory
err = os.RemoveAll(d.temp)
err := os.Rename(src, d.last)
if err != nil {
return err
}
@ -218,19 +258,26 @@ func (d *rbfFile) Close() error {
return nil
}
func copyFile(src, dest string) error {
input, err := ioutil.ReadFile(src)
from, err := os.Open(src)
if err != nil {
return err
}
defer from.Close()
err = ioutil.WriteFile(dest, input, 0644)
to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return err
}
defer to.Close()
_, err = io.Copy(to, from)
if err != nil {
return err
}
return nil
}
func Migrate(dataDir, backupPath string) error {
func Migrate(dataDir, backupPath string, verbose bool) error {
dataDir = strings.TrimSuffix(dataDir, "/")
err := os.MkdirAll(backupPath, 0777)
@ -279,7 +326,21 @@ func Migrate(dataDir, backupPath string) error {
bm := roaring.NewSliceBitmap()
for _, filename := range raw {
index, field, view, shard := Extract(filename)
sz, before := visited[filename]
fi, _ := os.Stat(dataDir + filename)
if field != "_exists" {
if !before {
visited[filename] = fi.Size()
} else {
if fi.Size() <= sz {
continue // skip it
}
visited[filename] = fi.Size()
}
}
if verbose {
glogger.Infof("processing: %v", dataDir+filename)
}
content, err := ioutil.ReadFile(dataDir + filename)
if err != nil {
return err
@ -293,35 +354,19 @@ func Migrate(dataDir, backupPath string) error {
if err != nil {
return err
}
tx, err := db.Begin(true)
if err != nil {
return err
}
key := string(txkey.Prefix(index, field, view, shard))
itr, ok := bm.Containers.Iterator(0)
if ok {
for itr.Next() {
k, v := itr.Value()
tx.PutContainer(key, k, v)
}
}
tx, err := db.Begin(true)
tx.AddRoaring(key, bm)
err = tx.Commit()
if err != nil {
return err
}
}
cache.Close()
keys := FetchIndexKeys(dataDir)
for _, filename := range keys {
fmt.Println("index keys", filename)
content, err := ioutil.ReadFile(filepath.Join(dataDir, filename))
if err != nil {
return err
}
glogger.Infof("index keys %v", filename)
srcFile := filepath.Join(dataDir, filename)
parts := strings.Split(filename, "/")
destFile := filepath.Join(backupPath, "indexes", parts[1], "translate", parts[3])
err = writeIfBigger(destFile, content)
err = writeIfBigger(destFile, srcFile)
if err != nil {
return err
}
@ -330,14 +375,11 @@ func Migrate(dataDir, backupPath string) error {
//deal with index field(row)keys
keys = FetchRowkeys(dataDir)
for _, filename := range keys {
fmt.Println("field", filename)
content, err := ioutil.ReadFile(dataDir + filename)
if err != nil {
return err
}
glogger.Infof("field %v", filename)
srcFile := dataDir + filename
parts := strings.Split(filename, "/")
destFile := filepath.Join(backupPath, "indexes", parts[1], "fields", parts[2], "translate")
err = writeIfBigger(destFile, content)
err = writeIfBigger(destFile, srcFile)
if err != nil {
return err
}
@ -345,16 +387,21 @@ func Migrate(dataDir, backupPath string) error {
return nil
}
func writeIfBigger(dst string, content []byte) error {
func writeIfBigger(dst string, srcFile string) error {
if stats, err := os.Stat(dst); os.IsNotExist(err) {
err = os.MkdirAll(filepath.Dir(dst), 0777)
if err != nil {
return err
}
return ioutil.WriteFile(dst, content, 0644)
return copyFile(srcFile, dst)
} else {
if stats.Size() < int64(len(content)) {
return ioutil.WriteFile(dst, content, 0644)
stats2, err := os.Stat(srcFile)
if err != nil {
return err
}
if stats.Size() < stats2.Size() {
vprint.VV("Bigger %v %v", stats.Size(), stats2.Size())
return copyFile(srcFile, dst)
}
}
return nil //simply skip it
@ -376,7 +423,7 @@ func FetchIndexKeys(base string) []string {
// first thing to do, check error. and decide what to do about it
if errX != nil {
fmt.Printf("error 「%v」 at a path 「%q」\n", errX, pathX)
glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX)
return errX
}
pathX = pathX[len(base):]
@ -393,7 +440,7 @@ func FetchIndexKeys(base string) []string {
err := filepath.Walk(base, ff)
if err != nil {
fmt.Printf("error walking the path %q: %v\n", base, err)
glogger.Errorf("error walking the path %q: %v\n", base, err)
}
return directory
}
@ -405,7 +452,7 @@ func FetchRowkeys(base string) []string {
// first thing to do, check error. and decide what to do about it
if errX != nil {
fmt.Printf("error 「%v」 at a path 「%q」\n", errX, pathX)
glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX)
return errX
}
pathX = pathX[len(base):]
@ -427,7 +474,7 @@ func FetchRowkeys(base string) []string {
err := filepath.Walk(base, ff)
if err != nil {
fmt.Printf("error walking the path %q: %v\n", base, err)
glogger.Errorf("error walking the path %q: %v\n", base, err)
}
return directory
}

56
cmd/roaring-migrate/main_test.go Normal file
View file

@ -0,0 +1,56 @@
package main
import (
"io/ioutil"
"os"
"testing"
)
func TestFileExists(t *testing.T) {
fileName := "missing"
if x, _ := fileExists(fileName); x {
t.Fatalf("file %v doesn't exist", fileName)
}
file, err := os.Create(fileName)
if err != nil {
t.Fatal(err)
}
file.Close()
if x, _ := fileExists(fileName); !x {
t.Fatalf("file %v doesn't exist", fileName)
}
t.Cleanup(func() {
os.Remove(fileName)
})
}
func TestMainProgram(t *testing.T) {
os.Args = []string{"roaring-migrate",
"--verbose",
}
if realMain() == 0 {
t.Fatal("should fail and it succeeded")
}
os.Args = []string{"roaring-migrate",
"--verbose",
}
if realMain() == 0 {
t.Fatal("should fail and it succeeded")
}
dir, err := ioutil.TempDir("", "backup")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir) // clean up
os.Args = []string{"roaring-migrate",
"--verbose=true",
"--data-dir=testdata/data-dir/",
"--backup-dir=" + dir,
}
if realMain() == 1 {
t.Fatal("shouldn't fail")
}
}

1
cmd/roaring-migrate/testdata/data-dir/.id vendored Normal file
View file

@ -0,0 +1 @@
6fc20f49-edf3-4211-8f6d-c670258ee6ea

1
cmd/roaring-migrate/testdata/data-dir/.startup.log vendored Normal file
View file

@ -0,0 +1 @@
2022-02-14T11:49:34.20065623-06:00 v2.7.0

2
cmd/roaring-migrate/testdata/data-dir/.topology vendored Normal file
View file

@ -0,0 +1,2 @@
$a317bd70-60ed-4723-99fa-3067563a708e$6fc20f49-edf3-4211-8f6d-c670258ee6ea

BIN
cmd/roaring-migrate/testdata/data-dir/idalloc.db vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/.data vendored Normal file
View file

Binary file not shown.

1
cmd/roaring-migrate/testdata/data-dir/repository/.meta vendored Normal file
View file

@ -0,0 +1 @@
 

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_exists/.data vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_exists/.meta vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_exists/keys vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222.cache vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/0 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/1 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/10 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/100 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/101 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/102 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/103 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/104 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/105 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/106 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/107 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/108 vendored Normal file
View file

Binary file not shown.

BIN
cmd/roaring-migrate/testdata/data-dir/repository/_keys/109 vendored Normal file
View file

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show more