diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index ce81cb4af..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,305 +0,0 @@ -# version: 2.1 - -# executors: -# golang: -# parameters: -# version: -# type: string -# default: "1.15.8" -# resource_class: -# type: string -# default: medium -# docker: -# - image: circleci/golang:<< parameters.version >> -# resource_class: << parameters.resource_class >> -# working_directory: /go/src/github.com/molecula/featurebase - -# commands: -# add-github-auth: -# steps: -# - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/" -# - run: git config --global url."https://${GITHUB_USER}:${GITHUB_PERSONAL_ACCESS_TOKEN}@github.com/".insteadOf "git@github.com:" -# restore-mod-cache: -# steps: -# - restore_cache: -# key: mod-cache-{{ checksum "go.sum" }} -# save-mod-cache: -# steps: -# - save_cache: -# key: mod-cache-{{ checksum "go.sum" }} -# paths: -# - /go/pkg/mod/ -# checkout-plus: -# steps: -# - add-github-auth -# - checkout -# - restore-mod-cache -# skip-if-root-unchanged: -# description: "skips the parent job if the PR includes no changes to featurebase" -# steps: -# - run: | -# ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true -# echo "ROOT_CHANGED_FILES = $ROOT_CHANGED_FILES" -# if [ -z "$ROOT_CHANGED_FILES" ] ; then -# echo "halting step" -# circleci step halt -# fi -# skip-if-lattice-unchanged: -# description: "skips the parent job if the PR includes no changes to lattice" -# steps: -# - run: | -# LATTICE_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep '^lattice/')" || true -# echo "LATTICE_CHANGED_FILES = $LATTICE_CHANGED_FILES" -# if [ -z "$LATTICE_CHANGED_FILES" ] ; then -# echo "halting step" -# circleci step halt -# fi - -# jobs: -# setup: -# executor: -# name: golang -# steps: -# - checkout-plus -# - run: go mod download -# - save-mod-cache -# linter: -# executor: -# name: golang -# steps: -# - checkout-plus -# - skip-if-root-unchanged -# - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.31.0 -# - run: make golangci-lint -# go-mod-tidy: -# executor: -# name: golang -# steps: -# - checkout-plus -# - skip-if-root-unchanged -# - run: go mod tidy -# - run: git diff --exit-code -- go.mod go.sum -# check-changelog-label: -# executor: -# name: golang -# steps: -# - run: '[[ -n $CIRCLE_PULL_REQUEST ]] || circleci step halt || true' # Skip if this is not a pull request -# - run: curl https://$GITHUB_USER:$GITHUB_PERSONAL_ACCESS_TOKEN@api.github.com/repos/molecula/featurebase/pulls/$(basename $CIRCLE_PULL_REQUEST) | jq "[.labels[] | .name | startswith(\"changelog\")] | any" -e -# test-build-arm: -# executor: -# name: golang -# steps: -# - checkout-plus -# - skip-if-root-unchanged -# - run: make build GOOS=linux GOARCH=arm GOARM=5 -# - run: make build GOOS=linux GOARCH=arm GOARM=6 -# - run: make build GOOS=linux GOARCH=arm GOARM=7 -# - run: make build GOOS=linux GOARCH=arm64 -# test: -# parameters: -# resource_class: -# type: string -# default: medium -# golang_version: -# type: string -# default: "1.15.8" -# shard_width: -# type: string -# default: "20" -# test_make_target: -# type: string -# default: "test" -# test_flags: -# type: string -# default: "" -# goarch: -# type: string -# default: amd64 -# executor: -# name: golang -# version: << parameters.golang_version >> -# resource_class: << parameters.resource_class >> -# environment: -# TMPDIR: /mnt/ramdisk -# steps: -# - checkout-plus -# - skip-if-root-unchanged -# - run: sudo apt-get update --allow-releaseinfo-change -y -# - run: sudo apt-get install lsof -# - run: -# command: make << parameters.test_make_target >> SHARD_WIDTH=<< parameters.shard_width >> GOARCH=<< parameters.goarch >> -# no_output_timeout: 30m -# test-external-lookup: -# docker: -# - image: circleci/golang:1.15.8 -# - image: circleci/postgres:13.2-ram -# environment: -# POSTGRES_PASSWORD=password -# steps: -# - checkout-plus -# - skip-if-root-unchanged -# - run: sudo apt-get update --allow-releaseinfo-change -y -# - run: sudo apt-get install postgresql-client -# - run: (for i in `seq 1 20`; do pg_isready -h localhost && exit 0 || sleep 1; done; exit 1) -# - run: -# command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable -# no_output_timeout: 30m -# cluster-tests: -# executor: -# name: golang -# steps: -# - checkout-plus -# - skip-if-root-unchanged -# - setup_remote_docker -# - run: make clustertests -# release: -# executor: -# name: golang -# steps: -# - checkout-plus -# - attach_workspace: -# at: . -# - setup_remote_docker: -# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 -# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin -# - run: make docker-release -# - store_artifacts: -# path: build -# - persist_to_workspace: -# root: . -# paths: build -# publish_release: -# executor: -# name: golang -# steps: -# - attach_workspace: -# at: . -# - run: go get github.com/tcnksm/ghr -# - run: ghr -t ${GITHUB_PERSONAL_ACCESS_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${CIRCLE_TAG} ./build/ -# docker-build: -# executor: -# name: golang -# steps: -# - checkout-plus -# - setup_remote_docker: -# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 -# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin -# - run: make docker GO_VERSION=1.15.8 -# - run: docker run featurebase:$(git describe --tags) help -# dockerhub-upload-unstable: -# executor: -# name: golang -# steps: -# - checkout-plus -# - setup_remote_docker: -# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 -# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin -# - run: make docker -# - run: docker run featurebase:$(git describe --tags) help -# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >> -# dockerhub-upload-stable: -# executor: -# name: golang -# steps: -# - checkout-plus -# - setup_remote_docker: -# version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 -# - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin -# - run: make docker -# - run: docker run featurebase:$(git describe --tags) help -# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >> -# - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest - -# workflows: -# build: -# jobs: -# - setup: -# context: molecula -# filters: -# tags: -# only: /^v.*/ -# - linter: -# context: molecula -# requires: -# - setup -# - go-mod-tidy: -# context: molecula -# requires: -# - setup -# - check-changelog-label: -# context: molecula -# requires: -# - setup -# - test-build-arm: -# context: molecula -# requires: -# - setup -# - test: -# name: test-golang-<< matrix.golang_version >> -# resource_class: large -# context: molecula -# requires: -# - setup -# matrix: -# parameters: -# golang_version: ["1.15.8", "1.16.10"] -# - test: -# name: << matrix.test_make_target >> -# resource_class: xlarge -# context: molecula -# requires: -# - setup -# matrix: -# parameters: -# test_make_target: ["test-race"] -# - test: -# name: test-shardwidth-22 -# context: molecula -# shard_width: "22" -# resource_class: large -# requires: -# - setup -# - test-external-lookup: -# context: molecula -# requires: -# - setup -# - cluster-tests: -# context: molecula -# requires: -# - setup -# - docker-build: -# context: molecula -# requires: -# - setup -# - release: -# context: molecula -# requires: -# - setup -# filters: -# tags: -# only: /^v.*/ -# - publish_release: -# context: molecula -# requires: -# - release -# filters: -# tags: -# only: /^v.*/ -# branches: -# ignore: /.*/ -# - dockerhub-upload-unstable: -# context: molecula -# requires: -# - setup -# filters: -# branches: -# only: master -# - dockerhub-upload-stable: -# context: molecula -# requires: -# - setup -# filters: -# tags: -# only: /^v.*/ -# branches: -# ignore: /.*/ diff --git a/.gitignore b/.gitignore index 675ffe9df..9e2547227 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ launch.json __pycache__/ report.xml outputs.json -builds/ \ No newline at end of file +builds/ +*.tfstate.backup \ No newline at end of file diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 729ff7f20..d5ecc1817 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -4,7 +4,7 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml variables: - GOVERSION: "1.16.13" + GOVERSION: "1.17.7" stages: - lint @@ -12,7 +12,9 @@ stages: - build - integration - gauntlet + - performance - post build + - nonblocking smoke build: image: golang:$GOVERSION @@ -34,6 +36,15 @@ golangci-lint: - echo "Checking for issues in new code" - golangci-lint run +go mod tidy: + stage: lint + image: golang:$GOVERSION + rules: + - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")' + script: + - go mod tidy + - git diff --exit-code -- go.mod go.sum + build lattice: stage: test image: node:14 @@ -83,11 +94,12 @@ run go tests: - aws run go tests race: - stage: test + stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests. image: golang:$GOVERSION rules: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' retry: 1 + needs: [] # don't wait to start running this. script: - echo "Running featurebase race tests..." - go test -race -v -timeout=90m ./... @@ -100,7 +112,7 @@ run go tests shardwidth22: rules: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - - echo "Running featurebase race tests..." + - echo "Running featurebase shardwidth22 tests..." - go test -timeout=30m -tags=shardwidth22 ./... tags: - aws @@ -287,6 +299,7 @@ clustertests: stage: integration tags: - shell + retry: 1 rules: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: @@ -301,6 +314,7 @@ authclustertests: variables: PROJECT: authclustertests_${CI_CONCURRENT_ID} stage: integration + retry: 1 tags: - shell rules: @@ -337,7 +351,6 @@ smoke test: AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY TF_VAR_cluster_prefix: "" - TF_VAR_branch: "" tags: - aws - docker @@ -369,8 +382,6 @@ smoke test: - export PATH=$PATH:/usr/local/go/bin - TF_VAR_cluster_prefix="smoke-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - TF_VAR_branch=$CI_COMMIT_BRANCH - - echo "Branch --> $TF_VAR_branch" script: - ./qa/scripts/setupSmokeTest.sh - ./qa/scripts/testSmokeTest.sh @@ -398,7 +409,6 @@ gauntlet: AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY ASG_NAME: "gitlab-runners" TF_VAR_cluster_prefix: "" - TF_VAR_branch: "" tags: - aws - docker @@ -433,8 +443,6 @@ gauntlet: - export PATH=$PATH:/usr/local/go/bin - TF_VAR_cluster_prefix="gauntlet-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" - - TF_VAR_branch=$CI_COMMIT_BRANCH - - echo "Branch --> $TF_VAR_branch" - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE script: @@ -455,7 +463,7 @@ s3 dump: tags: - shell rules: - - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' + - if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")' script: - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY @@ -482,3 +490,50 @@ s3 dump: - job: build for darwin arm64 - job: build for linux amd64 - job: build for linux arm64 + +perf_able: + stage: performance + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"' + trigger: + include: .gitlab/.perf-able-gitlab-ci.yml + variables: + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + +s3 dump tag: + stage: post build + variables: + PROFILE: "service-fb-ci" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + LOCATION: molecula-artifact-storage/featurebase/_tags + tags: + - shell + rules: + - if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web")' + script: + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY + - aws configure set region "us-east-2" + - aws configure set aws_profile $PROFILE + - | + for goos in "darwin" "linux"; do + for goarch in "amd64" "arm64"; do + dir=featurebase-${CI_COMMIT_TAG}-${goos}-${goarch} + echo "Directory ${dir}" + mkdir $dir + mv featurebase_${goos}_${goarch} ${dir}/featurebase + mv roaring-migrate_${goos}_${goarch} ${dir}/roaring-migrate + cp NOTICE install/featurebase.conf install/featurebase.*.service ${dir}/ + tar cvzf ${dir}.tar.gz ${dir} + aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive + aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/ + done + done + + needs: + - job: build for darwin amd64 + - job: build for darwin arm64 + - job: build for linux amd64 + - job: build for linux arm64 diff --git a/.gitlab/.perf-able-gitlab-ci.yml b/.gitlab/.perf-able-gitlab-ci.yml new file mode 100644 index 000000000..93165d50b --- /dev/null +++ b/.gitlab/.perf-able-gitlab-ci.yml @@ -0,0 +1,61 @@ +stages: + - performance + +perf_able: + stage: performance + timeout: 2h + image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-base:latest + variables: + PROFILE: "service-terraform" + INFRA_PROFILE: "service-gitlab" + AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY + AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY + ASG_NAME: "gitlab-runners" + TF_VAR_cluster_prefix: "" + tags: + - aws + - docker + - fbsmoke + before_script: + - apt-get update && apt-get install -y gnupg software-properties-common curl git + - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - + - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + - apt-get update && apt-get install terraform + - aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID --profile $PROFILE + - aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY --profile $PROFILE + - aws configure set region "us-east-2" --profile $PROFILE + - aws configure set aws_profile $PROFILE + - aws configure set aws_access_key_id $AWS_INFRA_ACCESS_KEY_ID --profile $INFRA_PROFILE + - aws configure set aws_secret_access_key $AWS_INFRA_SECRET_ACCESS_KEY --profile $INFRA_PROFILE + - aws configure set region "us-east-2" --profile $INFRA_PROFILE + - echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem + - chmod 400 gitlab-featurebase-ci.pem + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - mkdir -p ~/.ssh + - echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem + - chmod 400 /root/.ssh/gitlab-featurebase-ci.pem + - echo "$AWS_FBCI_SSH_KEY" | ssh-add - + - chmod 700 /root/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - apt update && apt -y install jq wget + - wget -q https://go.dev/dl/go1.17.5.linux-amd64.tar.gz + - tar -C /usr/local -xzf go1.17.5.linux-amd64.tar.gz + - export PATH=$PATH:/usr/local/go/bin + - TF_VAR_cluster_prefix="able-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + - echo "Cluster Prefix --> $TF_VAR_cluster_prefix" + - export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id) + - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE + script: + - ./qa/scripts/perf/able/ableSetup.sh + - ./qa/scripts/perf/able/ableTest.sh + after_script: + - ./qa/scripts/perf/able/ableTeardown.sh || true + - export INSTANCE_ID=$(cat instance_id) + - aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE + needs: + - pipeline: $PARENT_PIPELINE_ID + job: build for linux arm64 + - pipeline: $PARENT_PIPELINE_ID + job: build for linux amd64 \ No newline at end of file diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index be67afe75..a7a441112 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -1,7 +1,7 @@ # This Dockerfile is used for cluster testing - it produces a much larger image # and includes all of Go as well as some utilities. -FROM golang:1.16 +FROM golang:1.17 LABEL maintainer "dev@pilosa.com" diff --git a/Dockerfile-clustertests-client b/Dockerfile-clustertests-client index 2fb8cedf9..8e845a9f6 100644 --- a/Dockerfile-clustertests-client +++ b/Dockerfile-clustertests-client @@ -1,7 +1,7 @@ # This Dockerfile is used for cluster testing - it produces a much larger image # and includes all of Go as well as some utilities. -FROM golang:1.16 +FROM golang:1.17 LABEL maintainer "dev@pilosa.com" diff --git a/Makefile b/Makefile index a35dc42cd..fbd33204c 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) LDFLAGS="-X github.com/molecula/featurebase/v3.Version=$(VERSION) -X github.com/molecula/featurebase/v3.BuildTime=$(BUILD_TIME) -X github.com/molecula/featurebase/v3.Variant=$(VARIANT) -X github.com/molecula/featurebase/v3.Commit=$(COMMIT) -X github.com/molecula/featurebase/v3.TrialDeadline=$(TRIAL_DEADLINE)" -GO_VERSION=1.16.10 +GO_VERSION=1.17.7 DOCKER_BUILD= # set to 1 to use `docker-build` instead of `build` when creating a release BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia diff --git a/api.go b/api.go index 45892dce1..34155ee32 100644 --- a/api.go +++ b/api.go @@ -50,9 +50,6 @@ type API struct { importWorkerPoolSize int importWork chan importJob - usageCache *usageCache - schemaDetailsOn bool - Serializer Serializer } @@ -73,14 +70,6 @@ func OptAPIServer(s *Server) apiOption { } } -// Used to configure API option: schemaDetailsOn -func OptAPISchemaDetailsOn(isOn bool) apiOption { - return func(a *API) error { - a.schemaDetailsOn = isOn - return nil - } -} - func OptAPIImportWorkerPoolSize(size int) apiOption { return func(a *API) error { a.importWorkerPoolSize = size @@ -339,11 +328,17 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str } // Create field. - field, err := index.CreateFieldAndBroadcast(cfm) + field, err := index.CreateField(fieldName, opts...) if err != nil { return nil, errors.Wrap(err, "creating field") } + // Send the create field message to all nodes. We do this *outside* the + // CreateField logic so we're not blocking on it. + if err := api.holder.sendOrSpool(cfm); err != nil { + return nil, errors.Wrap(err, "sending CreateField message") + } + api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return field, nil } @@ -938,256 +933,6 @@ func (api *API) PrimaryNode() *topology.Node { return snap.PrimaryFieldTranslationNode() } -// Cache of disk usage statistics -type usageCache struct { - data map[string]NodeUsage - refreshInterval time.Duration - lastUpdated time.Time - resetTrigger chan bool - lastCalcDuration time.Duration - waitMultiplier float64 - disable bool - - muCalculate sync.Mutex - muAssign sync.Mutex -} - -var usageCacheMinDuration = 5 * time.Second // If usage takes less than this duration to calculate, don't use the cache. -var usageCacheMinInterval = time.Hour // Refresh interval is forced to be >= this duration. -var usageCacheInitialInterval = time.Hour // Refresh interval starts with this duration. - -// NodeUsage represents all usage measurements for one node. -type NodeUsage struct { - Disk DiskUsage `json:"diskUsage"` - Memory MemoryUsage `json:"memoryUsage"` - LastUpdated time.Time `json:"lastUpdated"` -} - -// DiskUsage represents the storage space used on disk by one node. -type DiskUsage struct { - Capacity uint64 `json:"capacity,omitempty"` - TotalUse uint64 `json:"totalInUse"` - IndexUsage map[string]IndexUsage `json:"indexes"` -} - -// IndexUsage represents the storage space used on disk by one index, on one node. -type IndexUsage struct { - Total uint64 `json:"total"` - IndexKeys uint64 `json:"indexKeys"` - FieldKeysTotal uint64 `json:"fieldKeysTotal"` - Fragments uint64 `json:"fragments"` - Metadata uint64 `json:"metadata"` - Fields map[string]FieldUsage `json:"fields"` -} - -// FieldUsage represents the storage space used on disk by one field, on one node -type FieldUsage struct { - Total uint64 `json:"total"` - Fragments uint64 `json:"fragments"` - Keys uint64 `json:"keys"` - Metadata uint64 `json:"metadata"` -} - -// MemoryUsage represents the memory used by one node. -type MemoryUsage struct { - Capacity uint64 `json:"capacity"` - TotalUse uint64 `json:"totalInUse"` -} - -// Returns disk usage from cache if cache is large. It will recalculate on the spot if the last cacluation was under 5 seconds. -func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, error) { - span, _ := tracing.StartSpanFromContext(ctx, "API.Usage") - defer span.Finish() - - if api.usageCache.disable { - resp := make(map[string]NodeUsage) - return resp, nil - } - - api.usageCache.muAssign.Lock() - lastCalc := api.usageCache.lastCalcDuration - api.usageCache.muAssign.Unlock() - if lastCalc < usageCacheMinDuration { - err := api.ResetUsageCache() - if err != nil { - api.server.logger.Infof("could not reset usageCache: %s", err) - } - } - - api.usageCache.muAssign.Lock() - lastUpdated := api.usageCache.lastUpdated - api.usageCache.muAssign.Unlock() - if lastUpdated == (time.Time{}) { - api.calculateUsage() - } - - if !remote { - api.requestUsageOfNodes() - } - - return api.usageCache.data, nil -} - -// Makes a ui/usage request for each node in cluster to calculates its usage and adds it to the cache -func (api *API) requestUsageOfNodes() { - nodes := api.cluster.Nodes() - for _, node := range nodes { - if node.ID == api.server.nodeID { - continue - } - - nodeUsage, err := api.server.defaultClient.GetNodeUsage(context.Background(), &node.URI) - if err != nil { - api.server.logger.Infof("couldn't collect disk usage from %s: %s", node.URI, err) - } - - api.usageCache.muAssign.Lock() - api.usageCache.data[node.ID] = nodeUsage[node.ID] - api.usageCache.muAssign.Unlock() - } -} - -// Calculates disk usage from scratch if cache has expired for each index and stores the results in the usage cache -func (api *API) calculateUsage() { - api.usageCache.muCalculate.Lock() - defer api.usageCache.muCalculate.Unlock() - api.server.wg.Add(1) - defer api.server.wg.Done() - - api.usageCache.muAssign.Lock() - lastUpdated := api.usageCache.lastUpdated - api.usageCache.muAssign.Unlock() - - if time.Since(lastUpdated) <= api.usageCache.refreshInterval { - return - } - indexDetails, nodeMetadataBytes, err := api.holder.Txf().IndexUsageDetails(api.isClosing) - if err != nil { - api.server.logger.Infof("couldn't get index usage details: %s", err) - } - if api.isClosing() { - return - } - - totalSize := nodeMetadataBytes - for _, s := range indexDetails { - totalSize += s.Total - } - - // NOTE: these errors are ignored in api.Info(), but checked here - si := api.server.systemInfo - diskCapacity, err := si.DiskCapacity(api.holder.path) - if err != nil { - api.server.logger.Infof("couldn't read disk capacity: %s", err) - } - - memoryCapacity, err := si.MemTotal() - if err != nil { - api.server.logger.Infof("couldn't read memory capacity: %s", err) - } - memoryUse, err := si.MemUsed() - if err != nil { - api.server.logger.Infof("couldn't read memory usage: %s", err) - } - - lastUpdated = time.Now() - // Insert into result. - nodeUsage := NodeUsage{ - Disk: DiskUsage{ - Capacity: diskCapacity, - TotalUse: totalSize, - IndexUsage: indexDetails, - }, - Memory: MemoryUsage{ - Capacity: memoryCapacity, - TotalUse: memoryUse, - }, - LastUpdated: lastUpdated, - } - api.usageCache.muAssign.Lock() - api.usageCache.data = make(map[string]NodeUsage) - api.usageCache.data[api.server.nodeID] = nodeUsage - api.usageCache.lastUpdated = lastUpdated - api.usageCache.muAssign.Unlock() -} - -// Periodically calculates disk/memory usage in terms of the duty cycle. The duty cycle represents the percentage of -// time that is spent recalculating this cache. It is specified relatively, rather than by a set interval, because -// scans can take an unpredictably long time. -func (api *API) RefreshUsageCache(dutyCycle float64) { - - if dutyCycle == 0 { - api.server.logger.Warnf("usage-duty-cycle set to 0, usage cache and /ui/usage endpoint are disabled") - api.usageCache = &usageCache{ - disable: true, - } - return - } - - trigger := make(chan bool) - defer close(trigger) - - multiplier := 100/dutyCycle - 1 - - api.usageCache = &usageCache{ - data: make(map[string]NodeUsage), - refreshInterval: usageCacheInitialInterval, - resetTrigger: trigger, - lastCalcDuration: 0, - waitMultiplier: multiplier, - } - api.server.logger.Infof("monitoring resource usage with duty cycle %v%%\n", dutyCycle) - for { - start := time.Now() - api.calculateUsage() - api.setRefreshInterval(time.Since(start)) - api.server.logger.Infof("updated resource usage cache at %v, took %v, next update in %v\n", api.usageCache.lastUpdated.Format(time.RFC3339), api.usageCache.lastCalcDuration.Truncate(time.Millisecond), api.usageCache.refreshInterval.Truncate(100*time.Millisecond)) - select { - case <-trigger: - continue - case <-api.server.closing: - return - case <-time.After(api.usageCache.refreshInterval): - continue - } - } -} - -// Refresh interval set in relation to how long the last calculation took. -func (api *API) setRefreshInterval(dur time.Duration) { - refresh := time.Duration(float64(dur) * api.usageCache.waitMultiplier) - if refresh < usageCacheMinInterval { - refresh = usageCacheMinInterval - } - api.usageCache.muAssign.Lock() - api.usageCache.refreshInterval = refresh - api.usageCache.lastCalcDuration = dur - api.usageCache.muAssign.Unlock() -} - -// Resets the lastUpdated time and awakens RefreshUsageCache() -func (api *API) ResetUsageCache() error { - if api.usageCache != nil { - api.usageCache.muAssign.Lock() - api.usageCache.lastUpdated = time.Time{} - api.usageCache.muAssign.Unlock() - } else { - return errors.New("invalidating cache: cache not initialized") - } - api.usageCache.resetTrigger <- true - return nil -} - -// isClosing returns true if the server is shutting down. -func (api *API) isClosing() bool { - select { - case <-api.server.closing: - return true - default: - return false - } -} - // RecalculateCaches forces all TopN caches to be updated. // This is done internally within a TopN query, but a user may want to do it ahead of time? func (api *API) RecalculateCaches(ctx context.Context) error { @@ -1272,38 +1017,6 @@ func (api *API) Schema(ctx context.Context, withViews bool) ([]*IndexInfo, error return api.holder.limitedSchema() } -// SchemaDetails returns information about each index in Pilosa including which -// fields they contain. Additional field information such as cardinality unless -// turned off via the schemaDetailsOn cli option. -func (api *API) SchemaDetails(ctx context.Context) ([]*IndexInfo, error) { - span, _ := tracing.StartSpanFromContext(ctx, "API.Schema") - defer span.Finish() - schema, err := api.holder.Schema() - if err != nil { - return nil, errors.Wrap(err, "getting schema") - } - if !api.schemaDetailsOn { - return schema, nil - } - for _, index := range schema { - for _, field := range index.Fields { - q := fmt.Sprintf("Count(Distinct(field=%s))", field.Name) - req := QueryRequest{Index: index.Name, Query: q} - resp, err := api.query(ctx, &req) - if err != nil { - return schema, errors.Wrapf(err, "querying cardinality (%s/%s)", index.Name, field.Name) - } - if len(resp.Results) == 0 { - continue - } - if card, ok := resp.Results[0].(uint64); ok { - field.Cardinality = &card - } - } - } - return schema, nil -} - // ApplySchema takes the given schema and applies it across the // cluster (if remote is false), or just to this node (if remote is // true). This is designed for the use case of replicating a schema @@ -1326,6 +1039,164 @@ func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error { return nil } +// applyOneIngestSchema applies a single ingestSpec, which specifies operations on +// a single index and possibly fields. If it is successful, it returns the name +// of the index and an empty slice (if it created the index), or the name of the +// index and a slice of the fields within that index that it created. If it +// is unsuccessful, it tries to delete whatever it created. +// +// The intended idiom is that if the returned list of fields isn't empty, the index +// already existed and only those fields need to be cleaned up in the event of +// a later error, but if the list of fields is empty, the entire index was new, +// and should be cleaned up, in which case there's no need to track or delete +// the specific fields separately. +func (api *API) ApplyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *Index, returnedFields []string, err error) { + if api.PrimaryNode().ID != api.NodeID() { + return nil, nil, RedirectError{ + HostPort: api.PrimaryNode().URI.Normalize(), + error: "request made to non-primary node", + } + } + + // create index + indexName := schema.IndexName + var createdFields []string + var useKeys bool + switch schema.PrimaryKeyType { + case "string": + useKeys = true + case "uint": + useKeys = false + default: + return nil, nil, fmt.Errorf("invalid primary key type %q", schema.PrimaryKeyType) + } + opts := IndexOptions{ + Keys: useKeys, + TrackExistence: true, + } + createdIndex := false + + // We check this up here because, if there's at least one field but we don't know what to do with + // it, we will necessarily fail, which means we'd delete the index anyway, so there's no point in + // trying to create it. We don't care about this if there's no fields specified. + if len(schema.Fields) > 0 { + switch schema.FieldAction { + case "create", "ensure", "require": + // do nothing + case "": + schema.FieldAction = schema.IndexAction + default: + return nil, nil, fmt.Errorf("invalid field-action %q, expecting create/ensure/require", schema.FieldAction) + } + } + + switch schema.IndexAction { + case "ensure", "require": + index, err = api.Index(ctx, indexName) + if err != nil { + if _, ok := err.(NotFoundError); !ok { + return nil, nil, fmt.Errorf("checking for existing index %q: %w", indexName, err) + } else { + err = nil + } + } + if index != nil { + existingOpts := index.Options() + if existingOpts != opts { + return nil, nil, fmt.Errorf("index %q options mismatch: schema %#v, existing %#v", indexName, opts, existingOpts) + } + break + } + if schema.IndexAction == "require" { + return nil, nil, fmt.Errorf("index %q does not exist", indexName) + } + fallthrough + case "create": + index, err = api.CreateIndex(ctx, indexName, opts) + if err != nil { + return nil, nil, err + } + createdIndex = true + default: + return nil, nil, fmt.Errorf("invalid index-action %q, need create/ensure/require", schema.IndexAction) + } + + // Now we might have an index, so we need our cleanup code. + defer func() { + if err == nil { + return + } + if createdIndex { + err := api.DeleteIndex(ctx, indexName) + if err != nil { + + api.server.logger.Printf("trying to undo failed index %q creation: %v", indexName, err) + } + return + } + for _, field := range createdFields { + err := api.DeleteField(ctx, indexName, field) + if err != nil { + api.server.logger.Printf("trying to undo failed field %q creation in index %q: %v", field, indexName, err) + } + } + }() + + // create all the fields specified in the index + for _, fSpec := range schema.Fields { + fieldName := fSpec.FieldName + opt := fieldSpecToFieldOption(fSpec) + err = opt.validate() + if err != nil { + return nil, nil, err + } + switch schema.FieldAction { + case "ensure", "require": + field, schemaErr := api.Field(ctx, indexName, fieldName) + if schemaErr != nil { + // NotFoundError is fine + if _, ok := schemaErr.(NotFoundError); !ok { + return nil, nil, fmt.Errorf("checking for existing field %q in %q: %w", fieldName, indexName, err) + } + } + if field != nil { + existing := field.Options() + if opt.Type != existing.Type { + return nil, nil, fmt.Errorf("existing field %q is %q, not %q", fieldName, existing.Type, opt.Type) + } + if ((opt.Keys != nil) && *opt.Keys) != existing.Keys { + if existing.Keys { + return nil, nil, fmt.Errorf("existing field %q in %q uses keys", fieldName, indexName) + } else { + return nil, nil, fmt.Errorf("existing field %q in %q doesn't use keys", fieldName, indexName) + } + } + // TODO: verify compatibility of other field opts, this is sorta hard + break + } + if schema.FieldAction == "require" { + return nil, nil, fmt.Errorf("field %q does not exist in %q", fieldName, indexName) + } + fallthrough + case "create": + fos := fieldOptionsToFunctionalOpts(opt) + _, err = api.CreateField(ctx, indexName, fieldName, fos...) + if err != nil { + return nil, nil, fmt.Errorf("creating field %q in %q: %v", fieldName, indexName, err) + } + createdFields = append(createdFields, fieldName) + } + } + + // we don't report the fields back, so we can distinguish "created index" + // from "created fields within index" + if createdIndex { + createdFields = nil + } + + return index, createdFields, nil +} + // Views returns the views in the given field. func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.Views") @@ -1905,6 +1776,13 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations") defer span.Finish() + if api.PrimaryNode().ID != api.NodeID() { + return RedirectError{ + HostPort: api.PrimaryNode().URI.Normalize(), + error: "request made to non-primary node", + } + } + if err := api.validate(apiIngestOperations); err != nil { return errors.Wrap(err, "validating api method") } @@ -3252,24 +3130,24 @@ var methodsResizing = map[apiMethod]struct{}{ apiSchema: {}, } -var methodsDegraded = map[apiMethod]struct{}{ - apiExportCSV: {}, - apiFragmentBlockData: {}, - apiFragmentBlocks: {}, - apiField: {}, - apiIndex: {}, - apiQuery: {}, - apiRecalculateCaches: {}, - apiRemoveNode: {}, - apiShardNodes: {}, - apiSchema: {}, - apiViews: {}, - apiStartTransaction: {}, - apiFinishTransaction: {}, - apiTransactions: {}, - apiGetTransaction: {}, - apiActiveQueries: {}, -} +// var methodsDegraded = map[apiMethod]struct{}{ +// apiExportCSV: {}, +// apiFragmentBlockData: {}, +// apiFragmentBlocks: {}, +// apiField: {}, +// apiIndex: {}, +// apiQuery: {}, +// apiRecalculateCaches: {}, +// apiRemoveNode: {}, +// apiShardNodes: {}, +// apiSchema: {}, +// apiViews: {}, +// apiStartTransaction: {}, +// apiFinishTransaction: {}, +// apiTransactions: {}, +// apiGetTransaction: {}, +// apiActiveQueries: {}, +// } var methodsNormal = map[apiMethod]struct{}{ apiCreateField: {}, diff --git a/api_test.go b/api_test.go index cd2595064..0ebf16b5f 100644 --- a/api_test.go +++ b/api_test.go @@ -29,6 +29,8 @@ import ( "github.com/molecula/featurebase/v3/shardwidth" "github.com/molecula/featurebase/v3/test" . "github.com/molecula/featurebase/v3/vprint" // nolint:staticcheck + + "golang.org/x/sync/errgroup" ) func TestAPI_Import(t *testing.T) { @@ -553,7 +555,7 @@ func TestAPI_Ingest(t *testing.T) { if err != nil { t.Fatalf("creating field: %v", err) } - _, err = coord.API.CreateField(ctx, index, timeField, pilosa.OptFieldTypeTime("YMD")) + _, err = coord.API.CreateField(ctx, index, timeField, pilosa.OptFieldTypeTime("YMD", "0")) if err != nil { t.Fatalf("creating field: %v", err) } @@ -676,7 +678,7 @@ func BenchmarkIngest(b *testing.B) { if err != nil { b.Fatalf("creating field: %v", err) } - _, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH")) + _, err = coord.API.CreateField(ctx, index, tqField, pilosa.OptFieldTypeTime("YMDH", "0")) if err != nil { b.Fatalf("creating field: %v", err) } @@ -956,29 +958,6 @@ func TestAPI_IDAlloc(t *testing.T) { }) } -func TestAPI_SchemaDetailsOff(t *testing.T) { - cluster := test.MustRunCluster(t, 2) - defer cluster.Close() - cmd := cluster.GetNode(0) - err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) - if err != nil { - t.Fatalf("could not toggle schema details to off: %v", err) - } - schema, err := cmd.API.SchemaDetails(context.Background()) - if err != nil { - t.Fatalf("getting schema: %v", err) - } - - for _, i := range schema { - for _, f := range i.Fields { - if f.Cardinality != nil { - t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality) - } - } - } - -} - type mutexCheckIndex struct { index *pilosa.Index indexName string @@ -1426,6 +1405,42 @@ func TestVariousApiTranslateCalls(t *testing.T) { } } +func TestAPI_CreateField(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := test.MustRunCluster(t, 3) + defer c.Close() + + nodes := make([]*test.Command, 3) + for i := range nodes { + nodes[i] = c.GetNode(i) + } + + if _, err := nodes[0].API.CreateIndex(ctx, "i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } + eg, ctx := errgroup.WithContext(context.Background()) + for _, n := range nodes { + node := n + eg.Go(func() error { + for i := 0; i < 10; i++ { + _, err := node.API.CreateField(ctx, "i", fmt.Sprintf("f%d", i)) + if err != nil && !errors.Is(err, pilosa.ErrFieldExists) { + return err + } + } + return nil + }) + } + err := eg.Wait() + if err != nil { + if errors.Is(err, pilosa.ErrFieldExists) { + t.Fatalf("conflict error: %v", err) + } + t.Fatalf("unexpected error: %T %v", err, err) + } +} + func TestAPI_RBFDebugInfo(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/apimethod_string.go b/apimethod_string.go index 11ed5a916..24703b05a 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -43,11 +43,15 @@ func _() { _ = x[apiIDReserve-32] _ = x[apiIDCommit-33] _ = x[apiIDReset-34] + _ = x[apiPartitionNodes-35] + _ = x[apiIngestOperations-36] + _ = x[apiIngestNodeOperations-37] + _ = x[apiMutexCheck-38] } -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDReset" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateIndexapiDeleteFieldapiDeleteAvailableShardapiDeleteIndexapiDeleteViewapiExportCSVapiFragmentBlockDataapiFragmentBlocksapiFragmentDataapiTranslateDataapiFieldTranslateDataapiFieldapiImportapiImportValueapiIndexapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiSchemaapiShardNodesapiStateapiViewsapiApplySchemaapiStartTransactionapiFinishTransactionapiTransactionsapiGetTransactionapiActiveQueriesapiPastQueriesapiIDReserveapiIDCommitapiIDResetapiPartitionNodesapiIngestOperationsapiIngestNodeOperationsapiMutexCheck" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 290, 304, 313, 326, 334, 342, 356, 375, 395, 410, 427, 443, 457, 469, 480, 490} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 82, 96, 109, 121, 141, 158, 173, 189, 210, 218, 227, 241, 249, 257, 277, 290, 304, 313, 326, 334, 342, 356, 375, 395, 410, 427, 443, 457, 469, 480, 490, 507, 526, 549, 562} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/boltdb/translate.go b/boltdb/translate.go index 8ff56f7e4..fe3d85d2c 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -12,7 +12,8 @@ import ( "sync" "time" - "github.com/molecula/featurebase/v3" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/roaring" "github.com/pkg/errors" bolt "go.etcd.io/bbolt" @@ -32,6 +33,8 @@ var ( bucketKeys = []byte("keys") bucketIDs = []byte("ids") + bucketFree = []byte("free") + freeKey = []byte("free") ) const ( @@ -119,6 +122,8 @@ func (s *TranslateStore) Open() (err error) { return err } else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil { return err + } else if _, err := tx.CreateBucketIfNotExists(bucketFree); err != nil { + return err } return nil }); err != nil { @@ -230,14 +235,26 @@ func (s *TranslateStore) CreateKeys(keys ...string) (map[string]uint64, error) { if idBucket == nil { return errors.Errorf(errFmtTranslateBucketNotFound, bucketIDs) } + freeBucket := tx.Bucket(bucketFree) + if freeBucket == nil { + return errors.Errorf(errFmtTranslateBucketNotFound, bucketFree) + } puts := 0 + + // we create a freeIDGetter to reduce marshalling + getter := newFreeIDGetter(freeBucket) + defer getter.Close() + for idx, key := range keys { id, boltKey := findIDByKey(keyBucket, key) if id != 0 { result[key] = id continue } - id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) + // see if we can re-use any IDs first + if id = getter.GetFreeID(); id == 0 { + id = pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) + } idBytes := idScratch[puts*8 : puts*8+8] binary.BigEndian.PutUint64(idBytes, id) puts++ @@ -498,6 +515,88 @@ func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error { } } +type boltWrapper struct { + tx *bolt.Tx + db *bolt.DB +} + +func (w *boltWrapper) Commit() error { + if w.tx != nil { + return w.tx.Commit() + } + return nil +} + +func (w *boltWrapper) Rollback() { + if w.tx != nil { + w.tx.Rollback() + } +} +func (s *TranslateStore) FreeIDs() (*roaring.Bitmap, error) { + result := roaring.NewBitmap() + err := s.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(bucketFree) + if bkt == nil { + return errors.Errorf(errFmtTranslateBucketNotFound, bucketKeys) + } + b := bkt.Get(freeKey) + err := result.UnmarshalBinary(b) + if err != nil { + return err + } + return nil + }) + return result, err +} +func (s *TranslateStore) MergeFree(tx *bolt.Tx, newIDs *roaring.Bitmap) error { + bkt := tx.Bucket(bucketFree) + b := bkt.Get(freeKey) + buf := new(bytes.Buffer) + if b != nil { //if existing combine with newIDs + before := roaring.NewBitmap() + err := before.UnmarshalBinary(b) + if err != nil { + return err + } + final := newIDs.Union(before) + _, err = final.WriteTo(buf) + if err != nil { + return err + } + } else { + newIDs.WriteTo(buf) + } + return bkt.Put(freeKey, buf.Bytes()) +} + +// Delete removes the lookeup pairs in order to make avialble for reuse but doesn't commit the +// transaction for that is tied to the associated rbf transaction being successful +func (s *TranslateStore) Delete(records *roaring.Bitmap) (pilosa.Commitor, error) { + tx, err := s.db.Begin(true) + if err != nil { + return nil, err + } + keyBucket := tx.Bucket(bucketKeys) + idBucket := tx.Bucket(bucketIDs) + ids := records.Slice() + for i := range ids { + id := u64tob(ids[i]) + boltKey := idBucket.Get(id) + err = keyBucket.Delete(boltKey) + if err != nil { + tx.Rollback() + return &boltWrapper{}, err + } + err = idBucket.Delete(id) + if err != nil { + tx.Rollback() + return &boltWrapper{}, err + } + + } + return &boltWrapper{tx: tx}, s.MergeFree(tx, records) +} + // emptyKey is a sentinel byte slice which stands for "" as a key. var emptyKey = []byte{ 0x00, 0x00, 0x00, @@ -521,6 +620,84 @@ func findIDByKey(bkt *bolt.Bucket, key string) (uint64, []byte) { return 0, boltKey } +// freeIDGetter reduces the amount of marshaling required to get multiple ids +type freeIDGetter struct { + freeBucket *bolt.Bucket + b *roaring.Bitmap + changed bool +} + +// newFreeIDGetter initializes a new freeIDGetter. If at any point there is a +// failure, it returns an error. +// +// NOTE: For changes to be persisted to the bucket, you must call +// (*freeIDGetter).Close() +func newFreeIDGetter(freeBucket *bolt.Bucket) *freeIDGetter { + g := &freeIDGetter{ + freeBucket: freeBucket, + } + // we ignore this value because it's okay if we dont have a bitmap just yet + _ = g.getBitmap() + return g +} + +func (g *freeIDGetter) getBitmap() bool { + if g.b == nil { + // get the bitmap from freeBucket + value := g.freeBucket.Get(freeKey) + if value == nil { + return false + } + // turn the value into a bitmap + b := roaring.NewBitmap() + if err := b.UnmarshalBinary(value); err != nil { + return false + } + g.b = b + } + return true +} + +// GetFreeID tries to get a free ID from the free id bucket. If at any point it +// fails to do so, it returns a 0. Otherwise, it returns the first free ID in the +// bucket +func (g *freeIDGetter) GetFreeID() (id uint64) { + if !g.getBitmap() { + return 0 + } + // get the first free id + id, ok := g.b.Min() + if !ok { + return 0 + } + // remove that id from the free id bitmap + if changed, err := g.b.RemoveN(id); changed == 0 || err != nil { + return 0 + } else { + g.changed = true + } + return id +} + +// Close persists any changes to the bitmap back to the bucket and then nils the +// references for safety. +func (g *freeIDGetter) Close() error { + if g.changed { + // convert bitmap to binary + buf, err := g.b.MarshalBinary() + if err != nil { + return errors.Wrap(err, "closing free ID Getter") + } + // put updated bitmap back into the freeBucket + if err := g.freeBucket.Put(freeKey, buf); err != nil { + return errors.Wrap(err, "closing free ID Getter") + } + } + g.b = nil + g.freeBucket = nil + return nil +} + func findKeyByID(bkt *bolt.Bucket, id uint64) string { boltKey := bkt.Get(u64tob(id)) if bytes.Equal(boltKey, emptyKey) { diff --git a/boltdb/translate_internal_test.go b/boltdb/translate_internal_test.go new file mode 100644 index 000000000..29d5c6fbb --- /dev/null +++ b/boltdb/translate_internal_test.go @@ -0,0 +1,107 @@ +package boltdb + +import ( + "path/filepath" + "testing" + + "github.com/molecula/featurebase/v3/roaring" + bolt "go.etcd.io/bbolt" +) + +func TestGetFreeID(t *testing.T) { + boltDir := t.TempDir() + db, err := bolt.Open(filepath.Join(boltDir, "testDB"), 0600, nil) + if err != nil { + t.Fatalf("unexpected error opening test boltdb: %v", err) + } + defer db.Close() + + makeTestBucket := func(tx *bolt.Tx, b *roaring.Bitmap) *bolt.Bucket { + if b == nil { + t.Fatalf("unexpected nil bitmap") + } + free, err := tx.CreateBucketIfNotExists(bucketFree) + if err != nil { + t.Fatalf("unexpected error making freeBucket: %v", err) + } + buf, err := b.MarshalBinary() + if err != nil { + t.Fatalf("unexpected error marshaling bitmap (%v) to binary: %v", b, err) + } + if err := free.Put(freeKey, buf); err != nil { + t.Fatalf("unexpected error adding data (%v) to freeBucket: %v", b, err) + } + return free + } + + for name, test := range map[string]struct { + bits *roaring.Bitmap + want uint64 + }{ + "bucket is there, but nobody's home": { + bits: roaring.NewBitmap(), + want: 0, + }, + "good bucket": { + bits: roaring.NewBitmap(1, 2, 34, 55, 9000), + want: 1, + }, + } { + t.Run(name, func(t *testing.T) { + tx, err := db.Begin(true) + if err != nil { + t.Fatalf("unexpected error starting bolt transaction: %v", err) + } + defer tx.Rollback() + freeBucket := makeTestBucket(tx, test.bits) + + getter := newFreeIDGetter(freeBucket) + defer getter.Close() + if got := getter.GetFreeID(); got != test.want { + t.Fatalf("expected %v got %v", test.want, got) + } + }) + } + + t.Run("CorrectOrdering", func(t *testing.T) { + tx, err := db.Begin(true) + if err != nil { + t.Fatalf("unexpected error starting bolt transaction: %v", err) + } + defer tx.Rollback() + + bucket := makeTestBucket(tx, roaring.NewBitmap(1, 34, 2, 55, 9000)) + + getter := newFreeIDGetter(bucket) + defer getter.Close() + for _, want := range []uint64{1, 2, 34, 55, 9000} { + if got := getter.GetFreeID(); got != want { + t.Fatalf("expected %v got %v", want, got) + } + } + if got := getter.GetFreeID(); got != 0 { + t.Fatalf("expected 0 got %v", got) + } + }) + + t.Run("NotABitmap", func(t *testing.T) { + tx, err := db.Begin(true) + if err != nil { + t.Fatalf("unexpected error starting bolt transaction: %v", err) + } + defer tx.Rollback() + + free, err := tx.CreateBucketIfNotExists(bucketFree) + if err != nil { + t.Fatalf("unexpected error making freeBucket: %v", err) + } + if err := free.Put(freeKey, []byte("this isn't right!")); err != nil { + t.Fatalf("unexpected error adding data to freeBucket: %v", err) + } + getter := newFreeIDGetter(free) + defer getter.Close() + if got := getter.GetFreeID(); got != 0 { + t.Fatalf("expected 0 got %v", got) + } + }) +} diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 201971644..cd74244eb 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -10,8 +10,9 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v3" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/boltdb" + "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/testhook" "github.com/molecula/featurebase/v3/topology" ) @@ -385,7 +386,53 @@ func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore { s.Path = f.Name() return s } +func TestTranslateStore_Delete(t *testing.T) { + s := MustOpenNewTranslateStore(t) + defer MustCloseTranslateStore(s) + // Setup initial keys. + ids, err := s.CreateKeys("foo", "bar", "deleteme") + if err != nil { + t.Fatal(err) + } + + records := roaring.NewBitmap(ids["deleteme"]) + c, err := s.Delete(records) + if err != nil { + t.Fatal(err) + } + if err = c.Commit(); err != nil { + t.Fatal(err) + } + r, e := s.FreeIDs() + if e != nil { + t.Fatal(err) + } + freeids := r.Slice() + if len(freeids) == 0 { + t.Fatalf("expected to have free id") + } + if freeids[0] != ids["deleteme"] { + t.Fatalf("expected [%v] and got %v", ids["deleteme"], freeids[0]) + } + + records2 := roaring.NewBitmap(ids["foo"]) + c, err = s.Delete(records2) + if err != nil { + t.Fatal(err) + } + if err = c.Commit(); err != nil { + t.Fatal(err) + } + r, e = s.FreeIDs() + if e != nil { + t.Fatal(err) + } + freeids = r.Slice() + if len(freeids) != 2 { + t.Fatalf("expected to have 2 free ids") + } +} func TestTranslateStore_ReadWrite(t *testing.T) { t.Run("WriteTo_ReadFrom", func(t *testing.T) { s := MustOpenNewTranslateStore(t) @@ -408,7 +455,7 @@ func TestTranslateStore_ReadWrite(t *testing.T) { // Put the contents of the store into a buffer. buf := bytes.NewBuffer(nil) - expN := int64(32768) + expN := s.Size() // After this, the buffer should contain batch0. if n, err := s.WriteTo(buf); err != nil { @@ -458,7 +505,7 @@ func TestTranslateStore_ReadWrite(t *testing.T) { func MustOpenNewTranslateStore(tb testing.TB) *boltdb.TranslateStore { s := MustNewTranslateStore(tb) if err := s.Open(); err != nil { - panic(err) + tb.Fatalf("opening s: %v", err) } return s } diff --git a/cache.go b/cache.go index 4dd5d5e4c..f558a28cf 100644 --- a/cache.go +++ b/cache.go @@ -194,6 +194,14 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) { } c.entries[id] = n + + // FB-1206: Periodically invalidate the cache when we are bulk loading + // as this can take up an upbounded amount of memory. This is especially + // true when restoring shards as all rows will be added. + if len(c.entries) > int(2*c.maxEntries) { + c.stats.Count(MetricRecalculateCache, 1, 1.0) + c.recalculate() + } } // Get returns a count for a given id. diff --git a/cache_test.go b/cache_test.go index 1ff8d0fff..0da077f1c 100644 --- a/cache_test.go +++ b/cache_test.go @@ -70,3 +70,15 @@ func TestCache_Rank_Dirty(t *testing.T) { t.Fatalf("wrote %v but got %v", expect, got) } } + +func TestCache_Rank_BulkAdd(t *testing.T) { + const cacheSize = 10 + cache := pilosa.NewRankCache(uint32(cacheSize)) + + for i := uint64(0); i < 1000; i++ { + cache.BulkAdd(i, i) + if n := cache.Len(); n > cacheSize*2 { + t.Fatalf("entry count exceed 2x cache size: %d", n) + } + } +} diff --git a/catcher.go b/catcher.go index a1f128d65..0a75ac9f7 100644 --- a/catcher.go +++ b/catcher.go @@ -26,6 +26,11 @@ func init() { var _ Tx = (*catcherTx)(nil) +func (c *catcherTx) RemoveChannel(index, field, view string, shard uint64, a chan uint64, resChan chan countResults) { + c.b.RemoveChannel(index, field, view, shard, a, resChan) + return +} + func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { return c.b.NewTxIterator(index, field, view, shard) } diff --git a/client/client.go b/client/client.go index 19d532e50..7e5796d74 100644 --- a/client/client.go +++ b/client/client.go @@ -832,31 +832,24 @@ func (c *Client) httpRequest(method string, path string, data []byte, headers ma body []byte err error ) - // try at most maxHosts non-failed hosts; protect against broken cluster.removeHost - for i := 0; i < maxHosts; i++ { + // try request on host, if it fails, try again on primary + for i := 0; i <= 1; i++ { host, herr := c.host(usePrimary) if herr != nil { return status, nil, errors.Wrapf(herr, "getting host, previous err: %v", err) } // doRequest implements expotential backoff status, body, err = c.doRequest(host, method, path, c.augmentHeaders(headers), data) - if err == nil { + // conditions when primary should not be tried + if err == nil || usePrimary || path == "/status" { break } - if c.manualServerURI == nil { - if usePrimary { - c.primaryLock.Lock() - c.primaryURI = nil - c.primaryLock.Unlock() - } else { - c.logger.Printf("removing host (%s) due to '%v'\n", host.Normalize(), err) - c.cluster.RemoveHost(host) - } - } + + usePrimary = true } if err != nil { - err = errors.Wrap(err, ErrTriedMaxHosts.Error()) + err = errors.Wrap(err, ErrHTTPRequest.Error()) } return status, body, err @@ -1674,17 +1667,18 @@ type SchemaField struct { // SchemaOptions contains options for a field or an index. type SchemaOptions struct { - FieldType FieldType `json:"type"` - CacheType string `json:"cacheType"` - CacheSize uint `json:"cacheSize"` - TimeQuantum string `json:"timeQuantum"` - Min pql.Decimal `json:"min"` - Max pql.Decimal `json:"max"` - Scale int64 `json:"scale"` - Keys bool `json:"keys"` - NoStandardView bool `json:"noStandardView"` - TrackExistence bool `json:"trackExistence"` - TimeUnit string `json:"timeUnit"` + FieldType FieldType `json:"type"` + CacheType string `json:"cacheType"` + CacheSize uint `json:"cacheSize"` + TimeQuantum string `json:"timeQuantum"` + Ttl time.Duration `json:"ttl"` + Min pql.Decimal `json:"min"` + Max pql.Decimal `json:"max"` + Scale int64 `json:"scale"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView"` + TrackExistence bool `json:"trackExistence"` + TimeUnit string `json:"timeUnit"` } func (so SchemaOptions) asIndexOptions() *IndexOptions { @@ -1702,6 +1696,7 @@ func (so SchemaOptions) asFieldOptions() *FieldOptions { cacheSize: int(so.CacheSize), cacheType: CacheType(so.CacheType), timeQuantum: TimeQuantum(so.TimeQuantum), + ttl: so.Ttl, min: so.Min, max: so.Max, scale: so.Scale, diff --git a/client/client_it_test.go b/client/client_it_test.go index ff8614d78..a58622bc8 100644 --- a/client/client_it_test.go +++ b/client/client_it_test.go @@ -497,7 +497,7 @@ func TestClientAgainstCluster(t *testing.T) { tmpcli, _ := NewClient(NewClusterWithHost(uri, uri, uri, uri), OptClientRetries(0)) _, err := tmpcli.Query(testIndex.All()) - require.Error(t, err, ErrTriedMaxHosts) + require.Error(t, err, ErrHTTPRequest) }) t.Run("InvalidQuery", func(t *testing.T) { diff --git a/client/cluster.go b/client/cluster.go index 0f1230583..cda424905 100644 --- a/client/cluster.go +++ b/client/cluster.go @@ -65,18 +65,6 @@ func (c *Cluster) Host() *pnet.URI { return host } -// RemoveHost black lists the host with the given pnet.URI from the cluster. -func (c *Cluster) RemoveHost(address *pnet.URI) { - c.mutex.Lock() - defer c.mutex.Unlock() - for i, uri := range c.hosts { - if uri.Equals(address) { - c.okList[i] = false - break - } - } -} - // Hosts returns all available hosts in the cluster. func (c *Cluster) Hosts() []pnet.URI { c.mutex.RLock() diff --git a/client/cluster_test.go b/client/cluster_test.go index 36790b7f8..53d63222a 100644 --- a/client/cluster_test.go +++ b/client/cluster_test.go @@ -49,22 +49,3 @@ func TestHosts(t *testing.T) { t.Fatalf("Host should return a value if there are hosts in the cluster") } } - -func TestRemoveHost(t *testing.T) { - uri, err := pnet.NewURIFromAddress("index1.pilosa.com:9999") - if err != nil { - t.Fatal(err) - } - c := NewClusterWithHost(uri) - if len(c.hosts) != 1 { - t.Fatalf("The cluster should contain the host") - } - uri, err = pnet.NewURIFromAddress("index1.pilosa.com:9999") - if err != nil { - t.Fatal(err) - } - c.RemoveHost(uri) - if len(c.Hosts()) != 0 { - t.Fatalf("The cluster should not contain the host") - } -} diff --git a/client/error.go b/client/error.go index 3c6685b62..f0fbb873f 100644 --- a/client/error.go +++ b/client/error.go @@ -12,7 +12,7 @@ var ( ErrInvalidFieldName = errors.New("Invalid field name") ErrInvalidLabel = errors.New("Invalid label") ErrInvalidKey = errors.New("Invalid key") - ErrTriedMaxHosts = errors.New("Tried max hosts, still failing") + ErrHTTPRequest = errors.New("Failed all HTTP retries") ErrAddrURIClusterExpected = errors.New("Addresses, URIs or a cluster is expected") ErrInvalidQueryOption = errors.New("Invalid query option") ErrInvalidIndexOption = errors.New("Invalid index option") diff --git a/client/ingest_api_batch_test.go b/client/ingest_api_batch_test.go index 9abfa15c6..bc6858c12 100644 --- a/client/ingest_api_batch_test.go +++ b/client/ingest_api_batch_test.go @@ -133,7 +133,6 @@ func TestIngestAPIBatchAdd(t *testing.T) { } func TestIngestAPIBatch(t *testing.T) { - t.Skip("causing sporadic CI failures... on my list to debug, but this code doesn't affect anyone's production anyhow (jaffee)") c := test.MustRunCluster(t, 3) defer c.Close() diff --git a/client/orm.go b/client/orm.go index ddb77bbd4..5035cc59c 100644 --- a/client/orm.go +++ b/client/orm.go @@ -728,6 +728,7 @@ type FieldInfo struct { type FieldOptions struct { fieldType FieldType timeQuantum TimeQuantum + ttl time.Duration cacheType CacheType cacheSize int min pql.Decimal @@ -751,6 +752,11 @@ func (fo FieldOptions) TimeQuantum() TimeQuantum { return fo.timeQuantum } +// Ttl returns the configured ttl for a time field. +func (fo FieldOptions) Ttl() time.Duration { + return fo.ttl +} + // CacheType returns the configured cache type for a "set" field. Empty string // otherwise. func (fo FieldOptions) CacheType() CacheType { @@ -826,6 +832,7 @@ func (fo FieldOptions) String() string { case FieldTypeTime: mopt["timeQuantum"] = string(fo.timeQuantum) mopt["noStandardView"] = fo.noStandardView + mopt["ttl"] = fo.ttl.String() case FieldTypeTimestamp: mopt["min"] = fo.min mopt["max"] = fo.max @@ -913,6 +920,12 @@ func OptFieldTypeTime(quantum TimeQuantum, opts ...bool) FieldOption { } } +func OptFieldTtl(dur time.Duration) FieldOption { + return func(options *FieldOptions) { + options.ttl = dur + } +} + // Timestamp field range. var ( DefaultEpoch = time.Unix(0, 0).UTC() // 1970-01-01T00:00:00Z diff --git a/client/orm_test.go b/client/orm_test.go index 650d113c3..29256a23c 100644 --- a/client/orm_test.go +++ b/client/orm_test.go @@ -925,7 +925,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), "", - "") + "", + 0) }) t.Run("IntFieldOptions", func(t *testing.T) { @@ -944,7 +945,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(-10, 0), pql.NewDecimal(100, 0), "", - "") + "", + 0) field = sampleIndex.Field("int-field2", OptFieldTypeInt(-10)) jsonString = field.options.String() @@ -962,7 +964,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(-10, 0), pql.NewDecimal(math.MaxInt64, 0), "", - "") + "", + 0) field = sampleIndex.Field("int-field3", OptFieldTypeInt()) jsonString = field.options.String() targetString = fmt.Sprintf(`{"options":{"type":"int","min":%d,"max":%d}}`, math.MinInt64, math.MaxInt64) @@ -978,7 +981,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(math.MinInt64, 0), pql.NewDecimal(math.MaxInt64, 0), "", - "") + "", + 0) field = sampleIndex.Field("int-field4", OptFieldTypeInt(), OptFieldForeignIndex("blerg")) jsonString = field.options.String() @@ -995,7 +999,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(math.MinInt64, 0), pql.NewDecimal(math.MaxInt64, 0), "blerg", - "") + "", + 0) }) t.Run("TimeFieldOptions", func(t *testing.T) { @@ -1004,7 +1009,7 @@ func TestORM(t *testing.T) { t.Fatalf("field noStandardView %v != %v", true, field.Opts().NoStandardView()) } jsonString := field.options.String() - targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH"}}` + targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH","ttl":"0s"}}` if sortedString(targetString) != sortedString(jsonString) { t.Fatalf("`%s` != `%s`", targetString, jsonString) } @@ -1017,7 +1022,31 @@ func TestORM(t *testing.T) { pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), "", - "") + "", + 0) + }) + + t.Run("TtlOptions", func(t *testing.T) { + field := sampleIndex.Field("ttl-field", OptFieldTypeTime(TimeQuantumDayHour, true), OptFieldTtl(0)) + if true != field.Opts().NoStandardView() { + t.Fatalf("field noStandardView %v != %v", true, field.Opts().NoStandardView()) + } + jsonString := field.options.String() + targetString := `{"options":{"noStandardView":true,"type":"time","timeQuantum":"DH","ttl":"0s"}}` + if sortedString(targetString) != sortedString(jsonString) { + t.Fatalf("`%s` != `%s`", targetString, jsonString) + } + compareFieldOptions(t, + field.Options(), + FieldTypeTime, + TimeQuantumDayHour, + CacheTypeDefault, + 0, + pql.NewDecimal(0, 0), + pql.NewDecimal(0, 0), + "", + "", + 0) }) t.Run("MutexFieldOptions", func(t *testing.T) { @@ -1036,7 +1065,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), "", - "") + "", + 0) }) t.Run("BoolFieldOptions", func(t *testing.T) { @@ -1055,7 +1085,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(0, 0), pql.NewDecimal(0, 0), "", - "") + "", + 0) }) t.Run("DecimalFieldOptions", func(t *testing.T) { @@ -1074,7 +1105,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(7, 3), pql.NewDecimal(999, 3), "", - "") + "", + 0) }) t.Run("DecimalFieldOptions", func(t *testing.T) { @@ -1093,7 +1125,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(7, 3), pql.NewDecimal(999, 3), "", - "") + "", + 0) }) t.Run("TimestampFieldOptions", func(t *testing.T) { @@ -1114,7 +1147,8 @@ func TestORM(t *testing.T) { pql.NewDecimal(MinTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0), pql.NewDecimal(MaxTimestamp.UnixNano()/TimeUnitNanos(pilosa.TimeUnitSeconds), 0), "", - pilosa.TimeUnitSeconds) + pilosa.TimeUnitSeconds, + 0) }) @@ -1163,7 +1197,7 @@ func comparePQL(t *testing.T, target string, q PQLQuery) { } } -func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, timeQuantum TimeQuantum, cacheType CacheType, cacheSize int, min pql.Decimal, max pql.Decimal, foreignIndex string, timeUnit string) { +func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, timeQuantum TimeQuantum, cacheType CacheType, cacheSize int, min pql.Decimal, max pql.Decimal, foreignIndex string, timeUnit string, ttl time.Duration) { if fieldType != opts.Type() { t.Fatalf("%s != %s", fieldType, opts.Type()) } @@ -1188,6 +1222,9 @@ func compareFieldOptions(t *testing.T, opts *FieldOptions, fieldType FieldType, if timeUnit != opts.TimeUnit() { t.Fatalf("%s != %s", timeUnit, opts.TimeUnit()) } + if ttl != opts.Ttl() { + t.Fatalf("%s != %s", ttl, opts.Ttl()) + } } func sortedString(s string) string { diff --git a/cmd/import.go b/cmd/import.go index d3f50dd27..77dfbb7f0 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -46,6 +46,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVar(&Importer.FieldOptions.CacheType, "field-cache-type", pilosa.CacheTypeRanked, "Specify the cache type for a set field on creation. One of: none, lru, ranked") flags.Uint32Var(&Importer.FieldOptions.CacheSize, "field-cache-size", 50000, "Specify the cache size for a set field on creation") flags.Var(&Importer.FieldOptions.TimeQuantum, "field-time-quantum", "Specify the time quantum for a time field on creation. One of: D, DH, H, M, MD, MDH, Y, YM, YMD, YMDH") + flags.DurationVarP(&Importer.FieldOptions.Ttl, "time-to-live", "t", 0, "Specify the time to live for views created by time quantum. Supported time unit: \"s\", \"m\", \"h\"") // \"ns\", \"us\" (or \"µs\"), \"ms\" also supported but ommitted for simplicity flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 72e6fa94a..129159f2b 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -13,25 +14,42 @@ import ( "syscall" pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/rbf" "github.com/molecula/featurebase/v3/rbf/cfg" "github.com/molecula/featurebase/v3/roaring" txkey "github.com/molecula/featurebase/v3/short_txkey" + "github.com/molecula/featurebase/v3/vprint" "github.com/spf13/cobra" ) +var visited map[string]int64 +var glogger = logger.NewStandardLogger(os.Stdout) + +const ( + Version = "1.0" +) + func main() { + os.Exit(realMain()) +} +func realMain() int { + visited = make(map[string]int64) var dataDir, backupPath string + var verbose bool cmdMigrate := &cobra.Command{ Use: "roaring-migrate", Short: "convert roaring pilosa backup to rbf", Long: `roaring-migrate uses the pilosa data-dir for each node, and produces a new backup that is able to be restored from utilizing the new pilosa restore tool.`, Run: func(cmd *cobra.Command, args []string) { + if verbose { + glogger.Infof("Version: %v", Version) + } nodes := strings.Split(dataDir, ",") for _, nodePath := range nodes { - err := Migrate(nodePath, backupPath) + err := Migrate(nodePath, backupPath, verbose) if err != nil { - fmt.Println("Error", err) + glogger.Errorf("%v", Version) return } @@ -40,24 +58,24 @@ func main() { } cmdMigrate.Flags().StringVarP(&dataDir, "data-dir", "d", "", "source directories for each node seperated by commas") cmdMigrate.Flags().StringVarP(&backupPath, "backup-dir", "b", "", "location of backup directory") + cmdMigrate.Flags().BoolVar(&verbose, "verbose", false, "additional progress information") err := cmdMigrate.MarkFlagRequired("data-dir") if err != nil { - fmt.Println("Error setting flag data-dir") - os.Exit(1) - return + glogger.Errorf("Error setting flag data-dir") + return 1 } err = cmdMigrate.MarkFlagRequired("backup-dir") if err != nil { - fmt.Println("Error setting flag backup-dir") - os.Exit(1) - return + glogger.Errorf("Error setting flag backup-dir") + return 1 } err = cmdMigrate.Execute() if err != nil { - fmt.Println("exec error", err) - os.Exit(1) + glogger.Errorf("exec error %v", err) + return 1 } + return 0 } func FetchFragments(base string) []string { @@ -67,7 +85,7 @@ func FetchFragments(base string) []string { // first thing to do, check error. and decide what to do about it if errX != nil { - fmt.Printf("error 「%v」 at a path 「%q」\n", errX, pathX) + glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX) return errX } pathX = pathX[len(base):] @@ -82,7 +100,7 @@ func FetchFragments(base string) []string { err := filepath.Walk(base, ff) if err != nil { - fmt.Printf("error walking the path %q: %v\n", base, err) + glogger.Errorf("error walking the path %q: %v\n", base, err) } return fragments } @@ -94,6 +112,14 @@ type local struct { Fields []*pilosa.FieldInfo `json:"fields,omitempty"` } +func fileExists(filename string) (bool, int64) { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false, 0 + } + return !info.IsDir(), info.Size() +} + func BuildSchema(dataDir string) ([]byte, error) { //need to find all the ".meta" files and load as field options @@ -105,7 +131,7 @@ func BuildSchema(dataDir string) ([]byte, error) { // first thing to do, check error. and decide what to do about it if errX != nil { - fmt.Printf("error 「%v」 at a path 「%q」\n", errX, pathX) + glogger.Infof("error 「%v」 at a path 「%q」\n", errX, pathX) return errX } pathX = pathX[len(dataDir):] @@ -115,7 +141,7 @@ func BuildSchema(dataDir string) ([]byte, error) { if strings.Contains(pathX, ".meta") { //convert the file to a fieldOptions // ex: metaPath /trait_store/aba/.meta - fmt.Println("PATHX", pathX) + glogger.Infof("PATHX %v", pathX) t := strings.Split(pathX, "/") index := t[1] src := dataDir + pathX @@ -160,7 +186,7 @@ func BuildSchema(dataDir string) ([]byte, error) { err := filepath.Walk(dataDir, ff) if err != nil { - fmt.Printf("error walking the path %q: %v\n", dataDir, err) + glogger.Errorf("error walking the path %q: %v\n", dataDir, err) } return json.MarshalIndent(schemaSerializer, "", " ") } @@ -183,7 +209,7 @@ func (d *rbfFile) getDB(path, index string, shard uint64) (*rbf.DB, error) { if d.last != src { d.Close() d.last = src - fmt.Println("RBF:", src) + glogger.Infof("RBF: %v", src) c := cfg.NewDefaultConfig() c.FsyncEnabled = false c.MinWALCheckpointSize = 0 @@ -197,20 +223,34 @@ func (d *rbfFile) getDB(path, index string, shard uint64) (*rbf.DB, error) { return d.working, nil } func (d *rbfFile) Close() error { + defer func() error { + // clean up the temp directory + err := os.RemoveAll(d.temp) + if err != nil { + return err + } + return nil + }() + if d.last != "" { d.working.Close() + //if d.last exists only keep the biggest - err := os.MkdirAll(filepath.Dir(d.last), 0777) - if err != nil { - return err + exists, sz := fileExists(d.last) + src := filepath.Join(d.temp, "data") + if !exists { + err := os.MkdirAll(filepath.Dir(d.last), 0777) + if err != nil { + return err + } + } else { + _, sz2 := fileExists(src) + if sz > sz2 { + return nil + } } // move the datafile backup shard - err = os.Rename(filepath.Join(d.temp, "data"), d.last) - if err != nil { - return err - } - //cleanup the tempdirectory - err = os.RemoveAll(d.temp) + err := os.Rename(src, d.last) if err != nil { return err } @@ -218,19 +258,26 @@ func (d *rbfFile) Close() error { return nil } func copyFile(src, dest string) error { - input, err := ioutil.ReadFile(src) + from, err := os.Open(src) if err != nil { return err } + defer from.Close() - err = ioutil.WriteFile(dest, input, 0644) + to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return err + } + defer to.Close() + + _, err = io.Copy(to, from) if err != nil { return err } return nil } -func Migrate(dataDir, backupPath string) error { +func Migrate(dataDir, backupPath string, verbose bool) error { dataDir = strings.TrimSuffix(dataDir, "/") err := os.MkdirAll(backupPath, 0777) @@ -279,7 +326,21 @@ func Migrate(dataDir, backupPath string) error { bm := roaring.NewSliceBitmap() for _, filename := range raw { index, field, view, shard := Extract(filename) - + sz, before := visited[filename] + fi, _ := os.Stat(dataDir + filename) + if field != "_exists" { + if !before { + visited[filename] = fi.Size() + } else { + if fi.Size() <= sz { + continue // skip it + } + visited[filename] = fi.Size() + } + } + if verbose { + glogger.Infof("processing: %v", dataDir+filename) + } content, err := ioutil.ReadFile(dataDir + filename) if err != nil { return err @@ -293,35 +354,19 @@ func Migrate(dataDir, backupPath string) error { if err != nil { return err } - tx, err := db.Begin(true) - if err != nil { - return err - } key := string(txkey.Prefix(index, field, view, shard)) - itr, ok := bm.Containers.Iterator(0) - if ok { - for itr.Next() { - k, v := itr.Value() - tx.PutContainer(key, k, v) - - } - } + tx, err := db.Begin(true) + tx.AddRoaring(key, bm) err = tx.Commit() - if err != nil { - return err - } } cache.Close() keys := FetchIndexKeys(dataDir) for _, filename := range keys { - fmt.Println("index keys", filename) - content, err := ioutil.ReadFile(filepath.Join(dataDir, filename)) - if err != nil { - return err - } + glogger.Infof("index keys %v", filename) + srcFile := filepath.Join(dataDir, filename) parts := strings.Split(filename, "/") destFile := filepath.Join(backupPath, "indexes", parts[1], "translate", parts[3]) - err = writeIfBigger(destFile, content) + err = writeIfBigger(destFile, srcFile) if err != nil { return err } @@ -330,14 +375,11 @@ func Migrate(dataDir, backupPath string) error { //deal with index field(row)keys keys = FetchRowkeys(dataDir) for _, filename := range keys { - fmt.Println("field", filename) - content, err := ioutil.ReadFile(dataDir + filename) - if err != nil { - return err - } + glogger.Infof("field %v", filename) + srcFile := dataDir + filename parts := strings.Split(filename, "/") destFile := filepath.Join(backupPath, "indexes", parts[1], "fields", parts[2], "translate") - err = writeIfBigger(destFile, content) + err = writeIfBigger(destFile, srcFile) if err != nil { return err } @@ -345,16 +387,21 @@ func Migrate(dataDir, backupPath string) error { return nil } -func writeIfBigger(dst string, content []byte) error { +func writeIfBigger(dst string, srcFile string) error { if stats, err := os.Stat(dst); os.IsNotExist(err) { err = os.MkdirAll(filepath.Dir(dst), 0777) if err != nil { return err } - return ioutil.WriteFile(dst, content, 0644) + return copyFile(srcFile, dst) } else { - if stats.Size() < int64(len(content)) { - return ioutil.WriteFile(dst, content, 0644) + stats2, err := os.Stat(srcFile) + if err != nil { + return err + } + if stats.Size() < stats2.Size() { + vprint.VV("Bigger %v %v", stats.Size(), stats2.Size()) + return copyFile(srcFile, dst) } } return nil //simply skip it @@ -376,7 +423,7 @@ func FetchIndexKeys(base string) []string { // first thing to do, check error. and decide what to do about it if errX != nil { - fmt.Printf("error 「%v」 at a path 「%q」\n", errX, pathX) + glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX) return errX } pathX = pathX[len(base):] @@ -393,7 +440,7 @@ func FetchIndexKeys(base string) []string { err := filepath.Walk(base, ff) if err != nil { - fmt.Printf("error walking the path %q: %v\n", base, err) + glogger.Errorf("error walking the path %q: %v\n", base, err) } return directory } @@ -405,7 +452,7 @@ func FetchRowkeys(base string) []string { // first thing to do, check error. and decide what to do about it if errX != nil { - fmt.Printf("error 「%v」 at a path 「%q」\n", errX, pathX) + glogger.Errorf("error 「%v」 at a path 「%q」\n", errX, pathX) return errX } pathX = pathX[len(base):] @@ -427,7 +474,7 @@ func FetchRowkeys(base string) []string { err := filepath.Walk(base, ff) if err != nil { - fmt.Printf("error walking the path %q: %v\n", base, err) + glogger.Errorf("error walking the path %q: %v\n", base, err) } return directory } diff --git a/cmd/roaring-migrate/main_test.go b/cmd/roaring-migrate/main_test.go new file mode 100644 index 000000000..8f452c178 --- /dev/null +++ b/cmd/roaring-migrate/main_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestFileExists(t *testing.T) { + fileName := "missing" + if x, _ := fileExists(fileName); x { + t.Fatalf("file %v doesn't exist", fileName) + } + file, err := os.Create(fileName) + if err != nil { + t.Fatal(err) + } + file.Close() + + if x, _ := fileExists(fileName); !x { + t.Fatalf("file %v doesn't exist", fileName) + } + + t.Cleanup(func() { + os.Remove(fileName) + }) +} + +func TestMainProgram(t *testing.T) { + os.Args = []string{"roaring-migrate", + "--verbose", + } + if realMain() == 0 { + t.Fatal("should fail and it succeeded") + } + os.Args = []string{"roaring-migrate", + "--verbose", + } + if realMain() == 0 { + t.Fatal("should fail and it succeeded") + } + dir, err := ioutil.TempDir("", "backup") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) // clean up + os.Args = []string{"roaring-migrate", + "--verbose=true", + "--data-dir=testdata/data-dir/", + "--backup-dir=" + dir, + } + if realMain() == 1 { + t.Fatal("shouldn't fail") + } + +} diff --git a/cmd/roaring-migrate/testdata/data-dir/.id b/cmd/roaring-migrate/testdata/data-dir/.id new file mode 100644 index 000000000..8b78590a2 --- /dev/null +++ b/cmd/roaring-migrate/testdata/data-dir/.id @@ -0,0 +1 @@ +6fc20f49-edf3-4211-8f6d-c670258ee6ea \ No newline at end of file diff --git a/cmd/roaring-migrate/testdata/data-dir/.startup.log b/cmd/roaring-migrate/testdata/data-dir/.startup.log new file mode 100644 index 000000000..704761102 --- /dev/null +++ b/cmd/roaring-migrate/testdata/data-dir/.startup.log @@ -0,0 +1 @@ +2022-02-14T11:49:34.20065623-06:00 v2.7.0 diff --git a/cmd/roaring-migrate/testdata/data-dir/.topology b/cmd/roaring-migrate/testdata/data-dir/.topology new file mode 100644 index 000000000..c434c3203 --- /dev/null +++ b/cmd/roaring-migrate/testdata/data-dir/.topology @@ -0,0 +1,2 @@ + +$a317bd70-60ed-4723-99fa-3067563a708e$6fc20f49-edf3-4211-8f6d-c670258ee6ea \ No newline at end of file diff --git a/cmd/roaring-migrate/testdata/data-dir/idalloc.db b/cmd/roaring-migrate/testdata/data-dir/idalloc.db new file mode 100644 index 000000000..e449c2898 Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/idalloc.db differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/.data b/cmd/roaring-migrate/testdata/data-dir/repository/.data new file mode 100644 index 000000000..efe3a38f1 Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/.data differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/.meta b/cmd/roaring-migrate/testdata/data-dir/repository/.meta new file mode 100644 index 000000000..af4e17a1c --- /dev/null +++ b/cmd/roaring-migrate/testdata/data-dir/repository/.meta @@ -0,0 +1 @@ +  \ No newline at end of file diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_exists/.data b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/.data new file mode 100644 index 000000000..efe3a38f1 Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/.data differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_exists/.meta b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/.meta new file mode 100644 index 000000000..2267b8ec2 Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/.meta differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_exists/keys b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/keys new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/keys differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222 b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222 new file mode 100644 index 000000000..ed0e9ad30 Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222.cache b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222.cache new file mode 100644 index 000000000..742c749a9 Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_exists/views/standard/fragments/222.cache differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/0 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/0 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/0 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/1 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/1 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/1 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/10 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/10 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/10 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/100 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/100 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/100 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/101 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/101 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/101 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/102 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/102 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/102 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/103 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/103 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/103 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/104 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/104 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/104 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/105 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/105 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/105 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/106 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/106 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/106 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/107 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/107 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/107 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/108 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/108 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/108 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/109 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/109 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/109 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/11 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/11 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/11 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/110 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/110 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/110 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/111 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/111 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/111 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/112 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/112 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/112 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/113 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/113 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/113 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/114 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/114 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/114 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/115 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/115 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/115 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/116 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/116 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/116 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/117 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/117 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/117 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/118 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/118 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/118 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/119 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/119 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/119 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/12 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/12 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/12 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/120 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/120 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/120 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/121 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/121 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/121 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/122 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/122 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/122 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/123 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/123 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/123 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/124 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/124 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/124 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/125 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/125 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/125 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/126 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/126 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/126 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/127 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/127 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/127 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/128 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/128 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/128 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/129 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/129 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/129 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/13 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/13 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/13 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/130 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/130 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/130 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/131 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/131 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/131 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/132 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/132 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/132 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/133 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/133 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/133 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/134 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/134 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/134 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/135 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/135 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/135 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/136 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/136 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/136 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/137 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/137 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/137 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/138 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/138 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/138 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/139 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/139 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/139 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/14 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/14 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/14 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/140 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/140 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/140 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/141 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/141 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/141 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/142 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/142 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/142 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/143 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/143 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/143 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/144 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/144 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/144 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/145 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/145 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/145 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/146 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/146 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/146 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/147 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/147 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/147 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/148 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/148 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/148 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/149 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/149 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/149 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/15 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/15 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/15 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/150 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/150 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/150 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/151 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/151 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/151 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/152 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/152 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/152 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/153 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/153 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/153 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/154 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/154 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/154 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/155 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/155 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/155 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/156 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/156 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/156 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/157 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/157 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/157 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/158 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/158 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/158 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/159 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/159 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/159 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/16 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/16 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/16 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/160 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/160 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/160 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/161 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/161 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/161 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/162 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/162 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/162 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/163 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/163 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/163 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/164 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/164 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/164 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/165 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/165 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/165 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/166 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/166 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/166 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/167 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/167 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/167 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/168 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/168 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/168 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/169 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/169 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/169 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/17 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/17 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/17 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/170 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/170 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/170 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/171 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/171 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/171 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/172 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/172 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/172 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/173 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/173 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/173 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/174 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/174 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/174 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/175 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/175 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/175 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/176 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/176 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/176 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/177 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/177 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/177 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/178 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/178 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/178 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/179 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/179 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/179 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/18 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/18 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/18 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/180 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/180 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/180 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/181 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/181 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/181 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/182 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/182 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/182 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/183 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/183 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/183 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/184 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/184 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/184 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/185 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/185 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/185 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/186 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/186 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/186 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/187 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/187 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/187 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/188 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/188 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/188 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/189 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/189 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/189 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/19 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/19 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/19 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/190 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/190 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/190 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/191 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/191 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/191 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/192 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/192 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/192 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/193 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/193 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/193 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/194 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/194 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/194 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/195 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/195 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/195 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/196 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/196 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/196 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/197 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/197 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/197 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/198 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/198 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/198 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/199 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/199 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/199 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/2 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/2 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/2 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/20 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/20 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/20 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/200 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/200 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/200 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/201 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/201 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/201 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/202 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/202 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/202 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/203 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/203 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/203 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/204 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/204 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/204 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/205 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/205 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/205 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/206 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/206 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/206 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/207 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/207 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/207 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/208 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/208 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/208 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/209 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/209 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/209 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/21 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/21 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/21 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/210 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/210 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/210 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/211 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/211 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/211 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/212 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/212 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/212 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/213 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/213 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/213 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/214 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/214 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/214 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/215 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/215 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/215 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/216 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/216 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/216 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/217 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/217 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/217 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/218 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/218 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/218 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/219 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/219 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/219 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/22 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/22 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/22 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/220 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/220 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/220 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/221 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/221 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/221 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/222 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/222 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/222 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/223 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/223 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/223 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/224 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/224 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/224 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/225 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/225 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/225 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/226 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/226 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/226 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/227 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/227 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/227 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/228 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/228 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/228 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/229 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/229 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/229 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/23 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/23 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/23 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/230 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/230 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/230 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/231 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/231 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/231 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/232 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/232 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/232 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/233 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/233 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/233 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/234 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/234 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/234 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/235 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/235 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/235 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/236 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/236 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/236 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/237 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/237 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/237 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/238 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/238 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/238 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/239 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/239 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/239 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/24 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/24 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/24 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/240 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/240 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/240 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/241 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/241 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/241 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/242 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/242 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/242 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/243 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/243 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/243 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/244 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/244 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/244 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/245 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/245 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/245 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/246 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/246 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/246 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/247 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/247 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/247 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/248 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/248 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/248 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/249 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/249 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/249 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/25 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/25 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/25 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/250 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/250 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/250 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/251 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/251 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/251 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/252 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/252 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/252 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/253 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/253 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/253 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/254 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/254 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/254 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/255 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/255 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/255 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/26 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/26 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/26 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/27 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/27 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/27 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/28 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/28 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/28 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/29 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/29 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/29 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/3 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/3 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/3 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/30 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/30 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/30 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/31 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/31 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/31 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/32 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/32 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/32 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/33 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/33 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/33 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/34 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/34 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/34 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/35 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/35 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/35 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/36 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/36 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/36 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/37 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/37 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/37 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/38 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/38 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/38 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/39 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/39 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/39 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/4 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/4 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/4 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/40 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/40 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/40 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/41 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/41 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/41 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/42 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/42 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/42 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/43 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/43 new file mode 100644 index 000000000..2deca814d Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/43 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/44 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/44 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/44 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/45 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/45 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/45 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/46 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/46 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/46 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/47 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/47 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/47 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/48 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/48 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/48 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/49 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/49 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/49 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/5 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/5 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/5 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/50 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/50 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/50 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/51 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/51 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/51 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/52 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/52 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/52 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/53 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/53 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/53 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/54 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/54 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/54 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/55 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/55 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/55 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/56 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/56 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/56 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/57 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/57 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/57 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/58 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/58 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/58 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/59 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/59 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/59 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/6 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/6 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/6 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/60 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/60 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/60 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/61 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/61 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/61 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/62 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/62 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/62 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/63 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/63 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/63 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/64 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/64 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/64 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/65 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/65 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/65 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/66 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/66 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/66 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/67 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/67 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/67 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/68 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/68 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/68 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/69 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/69 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/69 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/7 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/7 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/7 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/70 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/70 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/70 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/71 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/71 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/71 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/72 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/72 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/72 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/73 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/73 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/73 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/74 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/74 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/74 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/75 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/75 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/75 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/76 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/76 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/76 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/77 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/77 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/77 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/78 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/78 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/78 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/79 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/79 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/79 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/8 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/8 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/8 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/80 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/80 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/80 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/81 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/81 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/81 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/82 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/82 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/82 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/83 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/83 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/83 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/84 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/84 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/84 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/85 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/85 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/85 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/86 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/86 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/86 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/87 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/87 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/87 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/88 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/88 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/88 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/89 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/89 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/89 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/9 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/9 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/9 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/90 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/90 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/90 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/91 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/91 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/91 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/92 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/92 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/92 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/93 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/93 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/93 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/94 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/94 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/94 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/95 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/95 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/95 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/96 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/96 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/96 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/97 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/97 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/97 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/98 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/98 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/98 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/_keys/99 b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/99 new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/_keys/99 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/language/.data b/cmd/roaring-migrate/testdata/data-dir/repository/language/.data new file mode 100644 index 000000000..efe3a38f1 Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/language/.data differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/language/.meta b/cmd/roaring-migrate/testdata/data-dir/repository/language/.meta new file mode 100644 index 000000000..2267b8ec2 Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/language/.meta differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/language/keys b/cmd/roaring-migrate/testdata/data-dir/repository/language/keys new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/language/keys differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/language/views/standard/fragments/222 b/cmd/roaring-migrate/testdata/data-dir/repository/language/views/standard/fragments/222 new file mode 100644 index 000000000..f854440ff Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/language/views/standard/fragments/222 differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/language/views/standard/fragments/222.cache b/cmd/roaring-migrate/testdata/data-dir/repository/language/views/standard/fragments/222.cache new file mode 100644 index 000000000..160742634 --- /dev/null +++ b/cmd/roaring-migrate/testdata/data-dir/repository/language/views/standard/fragments/222.cache @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/stargazer/.data b/cmd/roaring-migrate/testdata/data-dir/repository/stargazer/.data new file mode 100644 index 000000000..efe3a38f1 Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/stargazer/.data differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/stargazer/.meta b/cmd/roaring-migrate/testdata/data-dir/repository/stargazer/.meta new file mode 100644 index 000000000..38cad5ccb Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/stargazer/.meta differ diff --git a/cmd/roaring-migrate/testdata/data-dir/repository/stargazer/keys b/cmd/roaring-migrate/testdata/data-dir/repository/stargazer/keys new file mode 100644 index 000000000..37e586e0e Binary files /dev/null and b/cmd/roaring-migrate/testdata/data-dir/repository/stargazer/keys differ diff --git a/ctl/server.go b/ctl/server.go index 2d8de2df2..497c0087c 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -90,15 +90,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)") flags.Uint16Var(&srv.Config.Postgres.SqlVersion, "postgres.sql-version", srv.Config.Postgres.SqlVersion, "Molecula Sql Handling Version (default 1)") - // Disk and Memory usage cache for ui/usage endpoint - flags.Float64Var(&srv.Config.UsageDutyCycle, "usage-duty-cycle", srv.Config.UsageDutyCycle, "Sets the percentage of time that is spent recalculating the disk and memory usage cache. 100.0 for always-running, 0 disables the cache and the /ui/usage endpoint.") - // Future flags. flags.BoolVar(&srv.Config.Future.Rename, "future.rename", false, "Present application name as FeatureBase. Defaults to false, will default to true in an upcoming release.") - // Toggle /schema/details endpoint. - flags.BoolVar(&srv.Config.SchemaDetailsOn, "schema-details-on", true, "Disable /schema/details endpoint") - // OAuth2.0 identity provider configuration flags.BoolVar(&srv.Config.Auth.Enable, "auth.enable", false, "Enable AuthN/AuthZ of featurebase, disabled by default.") flags.StringVar(&srv.Config.Auth.ClientId, "auth.client-id", srv.Config.Auth.ClientId, "Identity Provider's Application/Client ID.") diff --git a/delete_test.go b/delete_test.go index df977513b..62d39a789 100644 --- a/delete_test.go +++ b/delete_test.go @@ -8,7 +8,8 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v3" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/disco" "github.com/molecula/featurebase/v3/test" "github.com/stretchr/testify/require" ) @@ -33,7 +34,7 @@ func TestExecutor_DeleteRecords(t *testing.T) { {ID: 0, Val: 4}, {ID: 2, Val: 8}, }) - c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "timefield", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, "timefield", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) c.ImportBitsWithTimestamp(t, indexName, "timefield", [][2]uint64{ {0, 0}, {0, 1}, @@ -49,9 +50,24 @@ func TestExecutor_DeleteRecords(t *testing.T) { }) } + setupBig := func(t *testing.T, r *require.Assertions, c *test.Cluster, Rows uint64) { + t.Helper() + fieldName := "setfield" + c.CreateField(t, indexName, pilosa.IndexOptions{TrackExistence: true}, fieldName) + rows := make([][2]uint64, ShardWidth*Rows) + for columnID := uint64(0); columnID < ShardWidth; columnID++ { + for rowID := uint64(0); rowID < Rows; rowID++ { + if rowID == 0 || (columnID%rowID+1) != 0 { + rows[rowID] = [2]uint64{rowID, columnID} + } + } + } + c.ImportBits(t, indexName, "setfield", rows) + } + setupKeys := func(t *testing.T, r *require.Assertions, c *test.Cluster) { t.Helper() - c.CreateField(t, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "timefield", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.CreateField(t, indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "timefield", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) c.ImportTimeQuantumKey(t, indexName, "timefield", []test.TimeQuantumKey{ {RowKey: "fish", ColKey: "one", Ts: time.Date(2019, time.January, 2, 17, 45, 0, 0, time.UTC).Unix()}, {RowKey: "fish", ColKey: "one", Ts: time.Date(2020, time.January, 2, 17, 45, 0, 0, time.UTC).Unix()}, @@ -131,6 +147,12 @@ func TestExecutor_DeleteRecords(t *testing.T) { m = resp.Results[0].(pilosa.ExtractedTable) after := convertKey(m.Columns) require.Equal([]string{"B", "C", "D", "two"}, after, "these keyed records after delete") + //validate that column keys got deleted + node := c.GetNode(0) + keys := []string{"A", "one"} + res, err := node.API.FindIndexKeys(context.Background(), indexName, keys...) + require.Nil(err) + require.Empty(res) }) t.Run("Delete Row", func(t *testing.T) { setup(t, require, c) @@ -200,8 +222,33 @@ func TestExecutor_DeleteRecords(t *testing.T) { require.Equal([]uint64{0, 1}, after, "these records should be remaining") }) }) + t.Run("DeleteRecordsBigWithRestart", func(t *testing.T) { + c := test.MustNewCluster(t, 1) + for _, n := range c.Nodes { + n.Config.Cluster.ReplicaN = 1 + } + err := c.Start() + defer c.Close() + require.NoError(err, "Start cluster DeleteRecordsBig") + setupBig(t, require, c, 16) + defer tearDown(t, require, c) + node := c.GetNode(0) + resp := c.Query(t, indexName, `Delete(Row(setfield=12))`) + require.NotNil(resp, "Response should not be nil") + require.NotEmpty(resp.Results) + require.Equal(true, resp.Results[0], "Change should have happened") + resp = c.Query(t, indexName, `Count(Row(setfield=12))`) + require.NotNil(resp, "Response should not be nil") + require.NotEmpty(resp.Results) + require.Equal(uint64(0), resp.Results[0], "Should have removed") + err = node.Reopen() + require.NoError(err, "restart cluster DeleteRecordsBig") + err = c.AwaitState(disco.ClusterStateNormal, 10*time.Second) + require.NoError(err, "backToNormal") + }) } + func convert(before []pilosa.ExtractedTableColumn) []uint64 { result := make([]uint64, 0) for _, i := range before { diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index b9f826938..d0e894cfe 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -669,6 +669,7 @@ func (s Serializer) encodeFieldOptions(o *pilosa.FieldOptions) *pb.FieldOptions Scale: o.Scale, BitDepth: uint64(o.BitDepth), TimeQuantum: string(o.TimeQuantum), + Ttl: o.Ttl.String(), TimeUnit: string(o.TimeUnit), Keys: o.Keys, ForeignIndex: o.ForeignIndex, @@ -1072,6 +1073,11 @@ func (s Serializer) decodeFieldOptions(options *pb.FieldOptions, m *pilosa.Field m.Scale = options.Scale m.BitDepth = uint64(options.BitDepth) m.TimeQuantum = pilosa.TimeQuantum(options.TimeQuantum) + ttlValue, err := time.ParseDuration(options.Ttl) + if err != nil { + ttlValue = 0 + } + m.Ttl = ttlValue m.TimeUnit = options.TimeUnit m.Keys = options.Keys m.ForeignIndex = options.ForeignIndex diff --git a/executor.go b/executor.go index 232c1984c..136753a65 100644 --- a/executor.go +++ b/executor.go @@ -1529,6 +1529,8 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str Index: index, Field: fieldName, } + } else if field.Options().Type == FieldTypeTimestamp { + result = DistinctTimestamp{Name: fieldName} } else { result = SignedRow{} } @@ -1564,11 +1566,19 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str if err != nil { return nil, err } - results := make([]string, len(r.Pos.Columns())) - for i, val := range r.Pos.Columns() { + // If we have a filter, or there's just no content for this shard, we + // can end up with empty results. Rather than trying to synthesize + // a result from this empty set, we just go ahead and use that. + if r.Pos == nil { + return result, nil + } + cols := r.Pos.Columns() + results := make([]string, len(cols)) + for i, val := range cols { results[i] = FormatTimestampNano(int64(val), bsig.Base, field.options.TimeUnit) } - return DistinctTimestamp{Name: fieldName, Values: results}, nil + result = DistinctTimestamp{Name: fieldName, Values: results} + return result, nil } return executeDistinctShardBSI(ctx, qcx, idx, fieldName, shard, bsig, filterBitmap) } @@ -5725,7 +5735,7 @@ loop: continue loop } } - return nil, errors.Wrapf(errShardUnavailable, "%s:%d:%v:%v", index, shard, shards, nodes) + return nil, errors.Wrapf(errShardUnavailable, "%s:%d:%v", index, shard, nodes) } return m, nil } @@ -6721,6 +6731,17 @@ func (e *executor) translateCall(c *pql.Call, index string, columnKeys map[strin } } } + + // Check if "like" argument is applied to keyed fields. + if _, found := c.Args["like"].(string); found { + fieldName, err := c.FirstStringArg("_field", "field") + if err != nil || fieldName == "" { + return nil, fmt.Errorf("cannot read field name for Rows call") + } + if !idx.Field(fieldName).options.Keys { + return nil, fmt.Errorf("'%s' is not a set/mutex/time field with a string key", fieldName) + } + } } // Translate child calls. @@ -8242,70 +8263,128 @@ func (e *executor) executeDeleteRecords(ctx context.Context, qcx *Qcx, index str return n, nil } -func (e *executor) executeDeleteRecordFromShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (bool, error) { +func transactExistRow(ctx context.Context, idx *Index, shard uint64, frag *fragment, src *Row) (uint64, error) { + tx := idx.Txf().NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + rows, err := frag.rows(ctx, tx, 1) + if err != nil { + tx.Rollback() + return 0, err + } + rowID := uint64(len(rows) + 1) + _, err = frag.setRow(tx, src, rowID) + if err != nil { + tx.Rollback() + return 0, err + } + return rowID, tx.Commit() +} +func (e *executor) executeDeleteRecordFromShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (changed bool, err error) { span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeDeleteRecordFromShard") defer span.Finish() //need to build the bitmap in the call child := c.Children[0] - row, err := e.executeBitmapCallShard(ctx, qcx, index, child, shard) - if err != nil { - return false, err + src, er := e.executeBitmapCallShard(ctx, qcx, index, child, shard) + if er != nil { + err = er + return } - if len(row.segments) == 0 { //nothing to remove + if len(src.segments) == 0 { //nothing to remove + return + } + columns := src.segments[0].data //should only be one segment + if columns.Count() == 0 { + return + } + // Fetch index. + idx := e.Holder.Index(index) + if idx == nil { + err = newNotFoundError(ErrIndexNotFound, index) + return + } + + return DeleteRowsWithFlow(ctx, src, idx, shard, true) +} + +func DeleteRows(ctx context.Context, src *Row, idx *Index, shard uint64) (bool, error) { + return DeleteRowsWithFlow(ctx, src, idx, shard, false) +} +func DeleteRowsWithFlow(ctx context.Context, src *Row, idx *Index, shard uint64, normalFlow bool) (bool, error) { + var existenceFragment *fragment + var deletedRowID uint64 + var commitor Commitor = &NopCommitor{} + var err error + if len(src.segments) == 0 { //nothing to remove return false, nil } - columns := row.segments[0].data //should only be one segment + columns := src.segments[0].data //should only be one segment if columns.Count() == 0 { return false, nil } - // Fetch index. - idx := e.Holder.Index(index) - if idx == nil { - return false, newNotFoundError(ErrIndexNotFound, index) - } - - columnIDs := make([]uint64, 0) - none := make([]uint64, 0) // no bits will be set - - tx, finisher, err := qcx.GetTx(Txo{Write: writable, Index: idx, Shard: shard}) - if err != nil { - return false, err - } - defer finisher(&err) - changed := false - colCounts := make([]int, 0) - toClear := columnIDs[:0] - rowSet := make(map[uint64]struct{}) - callback := func(pos uint64) error { - toClear = append(toClear, pos) - rowID := pos / ShardWidth - rowSet[rowID] = struct{}{} - return nil - } - findExisting := roaring.NewBitmapBitmapFilter(columns, callback) - - clearFragment := func(frag *fragment) (bool, error) { - // re-zero these - toClear = columnIDs[:0] - rowSet = make(map[uint64]struct{}) - - err = tx.ApplyFilter(frag.index(), frag.field(), frag.view(), frag.shard, 0, findExisting) + if idx.Keys() { + //store columns in exits field ToBeDelete row commited + if normalFlow { // normalFlow is the standard path, "not normal" is recoverory + existenceFragment = idx.Holder().fragment(idx.Name(), existenceFieldName, viewStandard, shard) + if existenceFragment == nil { + //no exists field + return false, errors.New("can't bulk delete without existence field") + } + deletedRowID, err = transactExistRow(ctx, idx, shard, existenceFragment, src) + } + commitor, err = deleteKeyTranslation(ctx, idx, shard, columns) if err != nil { return false, err } - colCounts = append(colCounts, len(toClear)) - // this will be the remove part - if len(toClear) > 0 { - err = frag.importPositions(tx, none, toClear, rowSet) - if err != nil { - return false, err - } - return true, nil - } - return false, nil } + writeTx := idx.Txf().NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + if err != nil { + return false, err + } + defer writeTx.Rollback() + changed := false + defer func() { + //if there is an error on the bit clearing rollback the keys + if err != nil { + changed = false + commitor.Rollback() + return + } + // if there is an error in the key commit, then rollback the delete + // write records before keys to remove possiblity of unmatch keys=records + err = writeTx.Commit() + if err != nil { + changed = false + commitor.Rollback() + return + } + if er := commitor.Commit(); er != nil { + err = er + } + if err != nil { + idx.Holder().Logger.Errorf("problems committing delete in rbf %v shard %v", err, shard) + } + + }() + findExisting := roaring.NewBitmapBitmapFilter(columns, func(p uint64) error { return nil }) + resChan := make(chan countResults) + clearFragment := func(frag *fragment) (bool, error) { + posChan := make(chan uint64, 8192) + findExisting.SetCallback(func(pos uint64) error { + posChan <- pos + return nil + }) + go writeTx.RemoveChannel(frag.index(), frag.field(), frag.view(), frag.shard, posChan, resChan) + + err = writeTx.ApplyFilter(frag.index(), frag.field(), frag.view(), frag.shard, 0, findExisting) + close(posChan) + if err != nil { + return false, err + } + r := <-resChan + return r.changeCount > 0, r.err + } + for _, field := range idx.Fields() { for _, view := range field.views() { @@ -8320,7 +8399,44 @@ func (e *executor) executeDeleteRecordFromShard(ctx context.Context, qcx *Qcx, i if c { changed = true } + + } + } + close(resChan) + if existenceFragment != nil { //a string keys have been deleted and the deleteRow was created + if normalFlow { + existenceFragment.clearRow(writeTx, deletedRowID) + } else { + // this is if we are recovering from failure and cleaning up + rows, err := existenceFragment.rows(ctx, writeTx, 1) + if err != nil { + return false, err + } + for _, rowId := range rows { + existenceFragment.clearRow(writeTx, rowId) + } } } return changed, nil } + +type Commitor interface { + Rollback() + Commit() error +} +type NopCommitor struct { +} + +func (c *NopCommitor) Rollback() { + +} +func (c *NopCommitor) Commit() error { + return nil +} + +func deleteKeyTranslation(ctx context.Context, idx *Index, shard uint64, records *roaring.Bitmap) (Commitor, error) { + // ShardToShardParition ... + paritionID := topology.ShardToShardPartition(idx.name, shard, idx.holder.partitionN) + + return idx.TranslateStore(paritionID).Delete(records) +} diff --git a/executor_internal_test.go b/executor_internal_test.go index 171cb2ae3..0fa10f44e 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -545,3 +545,54 @@ func TestDistinctTimestampUnion(t *testing.T) { }) } } + +func TestExecutor_DeleteRows(t *testing.T) { + path, _ := testhook.TempDir(t, "pilosa-executor-") + holder := NewHolder(path, mustHolderConfig()) + defer holder.Close() + + if err := holder.Open(); err != nil { + t.Fatalf("opening holder: %v", err) + } + + idx, err := holder.CreateIndex("i", IndexOptions{TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + + f, err := idx.CreateField("f", OptFieldTypeDefault()) + if err != nil { + t.Fatalf("creating field: %v", err) + } + + shard := uint64(0) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + defer tx.Rollback() + + if _, err = f.SetBit(tx, 1, 1, nil); err != nil { + t.Fatalf("setting bit: %v", err) + } + + if err := tx.Commit(); err != nil { + t.Fatalf("failed to commit transaction: %v", err) + } + + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard}) + defer tx.Rollback() + + row, err := f.Row(tx, 1) + if err != nil { + t.Fatalf("failed to read row: %v", err) + } + + ctx := context.Background() + changed, err := DeleteRows(ctx, row, idx, shard) + if !changed || err != nil { + t.Fatalf("failed to delete row: %v", err) + } + + changed, err = DeleteRows(ctx, row, idx, shard) + if changed { + t.Fatalf("expected delete to not clear bit but it did") + } +} diff --git a/executor_test.go b/executor_test.go index f1da0402f..8064a2e5d 100644 --- a/executor_test.go +++ b/executor_test.go @@ -490,7 +490,7 @@ func TestExecutor(t *testing.T) { `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(c, t, writeQuery, readQueries, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { @@ -536,7 +536,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"six", "four", "five", "seven", "two", "three"}) { @@ -570,7 +570,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, nil, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"), pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { @@ -605,7 +605,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"), pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { @@ -639,7 +639,7 @@ func TestExecutor(t *testing.T) { `Row(f=1, from=946598400, to=1009854000)`, } responses := runCallTest(c, t, writeQuery, readQueries, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { @@ -674,7 +674,7 @@ func TestExecutor(t *testing.T) { `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(c, t, writeQuery, readQueries, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { @@ -693,7 +693,7 @@ func TestExecutor(t *testing.T) { `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, } responses = runCallTest(c, t, writeQuery, rq2, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("OldRange", func(t *testing.T) { if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) @@ -721,7 +721,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) t.Run("Standard", func(t *testing.T) { if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "seven", "four", "five", "six"}) { @@ -755,7 +755,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, nil, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"), pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { @@ -790,7 +790,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0"), pilosa.OptFieldKeys()) t.Run("Standard", func(t *testing.T) { @@ -990,7 +990,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, &pilosa.IndexOptions{TrackExistence: true}, - pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))) + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0")) if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } @@ -1059,7 +1059,7 @@ func TestExecutor(t *testing.T) { } responses := runCallTest(c, t, writeQuery, readQueries, - nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)) + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0", true)) for i := range responses { t.Run(fmt.Sprintf("response-%d", i), func(t *testing.T) { @@ -1730,7 +1730,7 @@ func TestExecutor_Execute_TopK_Time(t *testing.T) { defer c.Close() // Load some test data into a time field. - c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "f", pilosa.OptFieldTypeTime("YMD", true)) + c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "f", pilosa.OptFieldTypeTime("YMD", "0", true)) c.Query(t, "i", ` Set(0, f=0, 2016-01-02T00:00) Set(0, f=1, 2016-01-02T00:00) @@ -3413,7 +3413,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { }) t.Run("remote with timestamp", func(t *testing.T) { - _, err = c.GetPrimary().API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y")) + _, err = c.GetPrimary().API.CreateField(context.Background(), "i", "z", pilosa.OptFieldTypeTime("Y", "0")) if err != nil { t.Fatalf("creating field: %v", err) } @@ -3746,7 +3746,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { indexName := strings.ToLower(string(tt.quantum)) index := hldr.MustCreateIndexIfNotExists(indexName, pilosa.IndexOptions{}) // Create field. - if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum)); err != nil { + if _, err := index.CreateFieldIfNotExists("f", pilosa.OptFieldTypeTime(tt.quantum, "0")); err != nil { t.Fatal(err) } // Populate @@ -4691,14 +4691,14 @@ func TestExecutor_Execute_Extract(t *testing.T) { Set(3, keymutex="plugh") `) - c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "time", pilosa.OptFieldTypeTime("YMDH")) + c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "time", pilosa.OptFieldTypeTime("YMDH", "0")) c.Query(t, "i", ` Set(0, time=1, 2016-01-01T00:00) Set(1, time=2, 2017-01-01T00:00) Set(3, time=3, 2018-01-01T00:00) `) - c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keytime", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime("YMDH")) + c.CreateField(t, "i", pilosa.IndexOptions{TrackExistence: true}, "keytime", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime("YMDH", "0")) c.Query(t, "i", ` Set(0, keytime="h", 2016-01-01T00:00) Set(1, keytime="xyzzy", 2017-01-01T00:00) @@ -5087,7 +5087,7 @@ func TestExecutor_Execute_RowsTime(t *testing.T) { func TestExecutor_Execute_RowsTimeEmpty(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - c.CreateField(t, "i", pilosa.IndexOptions{}, "x", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)) + c.CreateField(t, "i", pilosa.IndexOptions{}, "x", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0", true)) rows := c.Query(t, "i", `Rows(x, from=1999-12-31T00:00, to=2002-01-01T03:00)`).Results[0].(pilosa.RowIdentifiers).Rows if !reflect.DeepEqual(rows, []uint64{}) { t.Fatalf("unexpected rows: %+v", rows) @@ -5452,6 +5452,11 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { t.Fatalf("creating field: %v", err) } + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f_id") + if err != nil { + t.Fatalf("creating field: %v", err) + } + // setup some data. 10 bits in each of shards 0 through 9. starting at // row/col shardNum and progressing to row/col shardNum+10. Also set the // previous 2 for each bit if row >0. @@ -5474,8 +5479,9 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { } tests := []struct { - q string - exp []string + q string + exp []string + expErr string }{ { q: `Rows(f)`, @@ -5557,13 +5563,26 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { q: `Rows(f, like="__")`, exp: []string{"10", "11", "12", "13", "14", "15", "16", "17", "18"}, }, + { + q: `Rows(f_id, like=7)`, + expErr: "parsing:", + }, + { + q: `Rows(f_id, like="__")`, + expErr: "executing: translating call:", + }, } for i, test := range tests { t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) { if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { - t.Fatal(err) + if !strings.HasPrefix(err.Error(), test.expErr) { + t.Fatal(err) + } } else { + if test.expErr != "" { + t.Fatalf("got success, expected error similar to: %+v", test.expErr) + } rows := res.Results[0].(pilosa.RowIdentifiers) if !reflect.DeepEqual(rows.Keys, test.exp) { t.Fatalf("\ngot: %+v\nexp: %+v", rows.Keys, test.exp) @@ -6789,13 +6808,16 @@ func variousQueriesCountDistinctTimestamp(t *testing.T, c *test.Cluster) { // create an index and timestamp field c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s")) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, "set") // add some data data := []string{"2010-01-02T12:32:00Z", "2010-04-20T12:32:00Z", "2011-04-20T12:59:00Z", "2011-04-20T12:40:00Z", "2011-04-20T12:32:00Z"} for i, datum := range data { - c.Query(t, index, fmt.Sprintf("Set(%d, ts=\"%s\")", i*(1<<20), datum)) + c.Query(t, index, fmt.Sprintf("Set(%d, ts=\"%s\")", i*ShardWidth, datum)) } + // set something in shard 8 so there's a shard present with no timestamp data + c.Query(t, index, fmt.Sprintf("Set(%d, set=0)", 8*ShardWidth)) // query the Count of Distinct vals in field ts count := c.Query(t, index, "Count(Distinct(field=ts))").Results[0] @@ -6803,6 +6825,13 @@ func variousQueriesCountDistinctTimestamp(t *testing.T, c *test.Cluster) { t.Fatalf("expected %v got %v", len(data), count) } + // query the ones that are in or after 2011, expecting 3. this helps us + // hit an edge case that only happens if you have no data *because of + // a filter*. + count = c.Query(t, index, "Count(Distinct(Row(ts > \"2011-01-01T00:00:00Z\"), field=ts))").Results[0] + if count != uint64(3) { + t.Fatalf("expected %v got %v", 3, count) + } } // Ensure that a top-level, bare distinct on multiple nodes @@ -7327,7 +7356,7 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) { // generic index // worth noting, since we are using YMDH resolution, both C4 & C5 // get binned to the same hour - c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f1", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f1", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) c.ImportTimeQuantumKey(t, "t_index", "f1", []test.TimeQuantumKey{ // from edge cases {ColKey: "C1", RowKey: "R1", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))}, @@ -7343,7 +7372,7 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) { }) // in this field, all columns have the same row value to simplify test queries for Row - c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f2", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + c.CreateField(t, "t_index", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "f2", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) c.ImportTimeQuantumKey(t, "t_index", "f2", []test.TimeQuantumKey{ // from {ColKey: "C1", RowKey: "R", Ts: ts(time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC))}, @@ -7493,7 +7522,7 @@ func populateTestData(t *testing.T, c *test.Cluster) { }) // Create and populate "places_visited" time field. - c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"))) + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"), "0")) ts2019Jan01 := int64(1546300800) * 1e+9 // 2019 January 1st 0:00:00 ts2019Aug01 := int64(1564617600) * 1e+9 // 2019 August 1st 0:00:00 ts2020Jan01 := int64(1577836800) * 1e+9 // 2020 January 1st 0:00:00 diff --git a/field.go b/field.go index 5beb2983a..228f18c8a 100644 --- a/field.go +++ b/field.go @@ -286,7 +286,7 @@ func OptFieldTypeDecimal(scale int64, minmax ...pql.Decimal) FieldOption { // used to specify the field as being type `time` and to // provide any respective configuration values. // Pass true to skip creation of the standard view. -func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption { +func OptFieldTypeTime(timeQuantum TimeQuantum, ttl string, opt ...bool) FieldOption { return func(fo *FieldOptions) error { if fo.Type != "" { return errors.Errorf("field type is already set to: %s", fo.Type) @@ -296,6 +296,11 @@ func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption { } fo.Type = FieldTypeTime fo.TimeQuantum = timeQuantum + ttlParsed, err := time.ParseDuration(ttl) + if err != nil { + return errors.Errorf("cannot parse ttl: %s", ttl) + } + fo.Ttl = ttlParsed fo.NoStandardView = len(opt) >= 1 && opt[0] return nil } @@ -674,6 +679,11 @@ func (f *Field) ForeignIndex() string { return f.options.ForeignIndex } +// Ttl returns the ttl of the field. +func (f *Field) Ttl() time.Duration { + return f.options.Ttl +} + func (f *Field) bitDepth() (uint64, error) { var maxBitDepth uint64 @@ -763,6 +773,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Base = 0 f.options.BitDepth = 0 f.options.TimeQuantum = "" + f.options.Ttl = 0 f.options.Keys = opt.Keys f.options.ForeignIndex = opt.ForeignIndex case FieldTypeInt, FieldTypeDecimal, FieldTypeTimestamp: @@ -776,6 +787,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.BitDepth = opt.BitDepth f.options.TimeUnit = opt.TimeUnit f.options.TimeQuantum = "" + f.options.Ttl = 0 f.options.Keys = opt.Keys f.options.ForeignIndex = opt.ForeignIndex @@ -809,6 +821,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { return ErrInvalidTimeQuantum } f.options.TimeQuantum = opt.TimeQuantum + f.options.Ttl = opt.Ttl f.options.ForeignIndex = opt.ForeignIndex case FieldTypeBool: f.options.Type = FieldTypeBool @@ -819,6 +832,7 @@ func (f *Field) applyOptions(opt FieldOptions) error { f.options.Base = 0 f.options.BitDepth = 0 f.options.TimeQuantum = "" + f.options.Ttl = 0 f.options.Keys = false f.options.ForeignIndex = "" default: @@ -1831,19 +1845,20 @@ func (p fieldInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FieldOptions represents options to set when initializing a field. type FieldOptions struct { - Base int64 `json:"base,omitempty"` - BitDepth uint64 `json:"bitDepth,omitempty"` - Min pql.Decimal `json:"min,omitempty"` - Max pql.Decimal `json:"max,omitempty"` - Scale int64 `json:"scale,omitempty"` - Keys bool `json:"keys"` - NoStandardView bool `json:"noStandardView,omitempty"` - CacheSize uint32 `json:"cacheSize,omitempty"` - CacheType string `json:"cacheType,omitempty"` - Type string `json:"type,omitempty"` - TimeUnit string `json:"timeUnit,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - ForeignIndex string `json:"foreignIndex"` + Base int64 `json:"base,omitempty"` + BitDepth uint64 `json:"bitDepth,omitempty"` + Min pql.Decimal `json:"min,omitempty"` + Max pql.Decimal `json:"max,omitempty"` + Scale int64 `json:"scale,omitempty"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView,omitempty"` + CacheSize uint32 `json:"cacheSize,omitempty"` + CacheType string `json:"cacheType,omitempty"` + Type string `json:"type,omitempty"` + TimeUnit string `json:"timeUnit,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + ForeignIndex string `json:"foreignIndex"` + Ttl time.Duration `json:"ttl,omitempty"` } // newFieldOptions returns a new instance of FieldOptions @@ -1954,15 +1969,17 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { }) case FieldTypeTime: return json.Marshal(struct { - Type string `json:"type"` - TimeQuantum TimeQuantum `json:"timeQuantum"` - Keys bool `json:"keys"` - NoStandardView bool `json:"noStandardView"` + Type string `json:"type"` + TimeQuantum TimeQuantum `json:"timeQuantum"` + Keys bool `json:"keys"` + NoStandardView bool `json:"noStandardView"` + Ttl time.Duration `json:"Ttl"` }{ o.Type, o.TimeQuantum, o.Keys, o.NoStandardView, + o.Ttl, }) case FieldTypeMutex: return json.Marshal(struct { diff --git a/field_internal_test.go b/field_internal_test.go index 98dea321b..5f0f7d013 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -343,7 +343,7 @@ func TestField_CreateViewIfNotExists(t *testing.T) { } func TestField_SetTimeQuantum(t *testing.T) { - f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"))) + f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"), "0")) defer f.Close() // Retrieve time quantum. @@ -360,7 +360,7 @@ func TestField_SetTimeQuantum(t *testing.T) { } func TestField_RowTime(t *testing.T) { - f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"))) + f := OpenField(t, OptFieldTypeTime(TimeQuantum("YMDH"), "0")) defer f.Close() // Obtain transaction. diff --git a/fragment.go b/fragment.go index 54d68eed9..c02e24699 100644 --- a/fragment.go +++ b/fragment.go @@ -770,50 +770,31 @@ func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint64, val // sum returns the sum of a given bsiGroup as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. func (f *fragment) sum(tx Tx, filter *Row, bitDepth uint64) (sum int64, count uint64, err error) { - // Compute count based on the existence row. - consider, err := f.row(tx, bsiExistsBit) - if err != nil { - return sum, count, err - } else if filter != nil { - consider = consider.Intersect(filter) - } - count = consider.Count() - - // Get negative set - nrow, err := f.row(tx, bsiSignBit) - if err != nil { - return sum, count, err - } - - // Filter negative set - nrow = consider.Intersect(nrow) - - // Get postive set - prow := consider.Difference(nrow) - - // Compute the sum based on the bit count of each row multiplied by the - // place value of each row. For example, 10 bits in the 1's place plus - // 4 bits in the 2's place plus 3 bits in the 4's place equals a total - // sum of 30: - // - // 10*(2^0) + 4*(2^1) + 3*(2^2) = 30 - // - // Execute once for positive numbers and once for negative. Subtract the - // negative sum from the positive sum. - for i := uint64(0); i < bitDepth; i++ { - row, err := f.row(tx, uint64(bsiOffsetBit+i)) - if err != nil { - return sum, count, err + // If there's a provided filter, but it has no contents for this particular + // shard, we're done and can return early. If there's no provided filter, + // though, we want to run with no-filter, as opposed to an empty filter. + var filterData *roaring.Bitmap + if filter != nil { + for _, seg := range filter.segments { + if seg.shard == f.shard { + filterData = seg.data + break + } } - - psum := int64((1 << i) * row.intersectionCount(prow)) - nsum := int64((1 << i) * row.intersectionCount(nrow)) - - // Squash to reduce the possibility of overflow. - sum += psum - nsum + // if filter is empty, we're done + if filterData == nil { + return 0, 0, nil + } + } + bsiFilt := roaring.NewBitmapBSICountFilter(filterData) + err = tx.ApplyFilter(f.index(), f.field(), f.view(), f.shard, 0, bsiFilt) + if err != nil && err != io.EOF { + return sum, count, errors.Wrap(err, "finding existing positions") } - return sum, count, nil + c32, sum := bsiFilt.Total() + + return sum, uint64(c32), nil } // min returns the min of a given bsiGroup as well as the number of columns involved. diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 6fd803954..3beea99ec 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1570,6 +1570,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // make a read-only Tx after ReadFrom has committed. tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f1, Shard: f1.shard}) + defer tx.Rollback() // Verify cache is in other fragment. if n := f1.cache.Len(); n != 1 { diff --git a/go.mod b/go.mod index 529f2a29d..4d1f6361d 100644 --- a/go.mod +++ b/go.mod @@ -54,11 +54,12 @@ require ( go.etcd.io/etcd v0.0.0-20201125193152-8a03d2e9614b golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 // indirect golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7 - golang.org/x/mod v0.4.2 - golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect + golang.org/x/mod v0.5.1 + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect + golang.org/x/text v0.3.7 // indirect google.golang.org/grpc v1.28.0 gopkg.in/yaml.v2 v2.4.0 modernc.org/mathutil v1.0.0 diff --git a/go.sum b/go.sum index 79fc9f1f6..c89a96884 100644 --- a/go.sum +++ b/go.sum @@ -444,8 +444,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -465,8 +465,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -515,8 +515,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= diff --git a/hack.go b/hack.go index 4f8a8a460..eb1f5787b 100644 --- a/hack.go +++ b/hack.go @@ -2,6 +2,8 @@ package pilosa import ( + "time" + "github.com/gogo/protobuf/proto" "github.com/molecula/featurebase/v3/pb" "github.com/molecula/featurebase/v3/pql" @@ -57,6 +59,11 @@ func UnmarshalFieldOptions(name string, createdAt int64, buf []byte) (*FieldInfo fi.Options.Base = pbi.Base fi.Options.BitDepth = pbi.BitDepth fi.Options.TimeQuantum = TimeQuantum(pbi.TimeQuantum) + ttlValue, err := time.ParseDuration(pbi.Ttl) + if err != nil { + ttlValue = 0 + } + fi.Options.Ttl = ttlValue fi.Options.Keys = pbi.Keys fi.Options.NoStandardView = pbi.NoStandardView diff --git a/holder.go b/holder.go index db38a6db7..4e9030f40 100644 --- a/holder.go +++ b/holder.go @@ -287,6 +287,50 @@ func (h *Holder) IndexesPath() string { return filepath.Join(h.path, IndexesDir) } +// processDeleteInflight checks if deletion was in progress when server shutdown +// the _exists field is set to row+1 when delete is started. Upon completion, the row is deleted. +// if _exists>=1, we finish deleting the rows +func (h *Holder) processDeleteInflight() error { + for _, index := range h.Indexes() { + if index.trackExistence { + shards := index.AvailableShards(includeRemote).Slice() + + for _, shard := range shards { + inprocessRowIDs := NewRow() + + frag := h.fragment(index.name, existenceFieldName, viewStandard, shard) + if frag == nil { + continue + } + + tx := index.Txf().NewTx(Txo{Write: !writable, Index: index, Shard: shard}) + defer tx.Rollback() + + // filter rows based on having _exists>=1, which is used to flag delete in-flight + rows, err := frag.rows(context.Background(), tx, 1) + if err != nil { + return err + } + + // check if any rows are found + if len(rows) == 0 { + return nil + } + + for _, rowID := range rows { + row, err2 := frag.row(tx, rowID) + if err2 != nil { + return err2 + } + inprocessRowIDs = inprocessRowIDs.Union(row) + } + DeleteRows(context.Background(), inprocessRowIDs, index, shard) + } + } + } + return nil +} + // Open initializes the root data directory for the holder. func (h *Holder) Open() error { h.opening = true @@ -380,6 +424,9 @@ func (h *Holder) Open() error { return errors.Wrap(err, "processing foreign index fields") } + // Check if deletion was in progress when server was shutdown + h.processDeleteInflight() + h.Stats.Open() h.opened.Close() diff --git a/holder_internal_test.go b/holder_internal_test.go index c9e164f27..2fec338af 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -2,7 +2,10 @@ package pilosa import ( + "testing" + "github.com/molecula/featurebase/v3/disco" + "github.com/molecula/featurebase/v3/testhook" ) // mustHolderConfig sets up a default holder config for tests. @@ -14,3 +17,74 @@ func mustHolderConfig() *HolderConfig { cfg.Sharder = disco.InMemSharder return cfg } + +func TestHolder_ProcessDeleteInflight(t *testing.T) { + path, _ := testhook.TempDir(t, "delete-inflight") + h := NewHolder(path, mustHolderConfig()) + defer h.Close() + + err := h.Open() + if err != nil { + t.Fatalf("failed to open holder: %v", err) + } + + idx, err := h.CreateIndexIfNotExists("i", IndexOptions{TrackExistence: true}) + if err != nil { + t.Fatalf("failed to create index: %v", err) + } + f, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault()) + if err != nil { + t.Fatalf("failed to create field: %v", err) + } + + existencefield := idx.existenceFld + shard := uint64(0) + tx := idx.Txf().NewTx(Txo{Write: true, Index: idx, Shard: shard}) + defer tx.Rollback() + + rowCol := []struct { + row uint64 + col uint64 + }{ + {1, 1}, + {1, 2}, + {30, 33}, + {22, 2}, + } + for _, r := range rowCol { + _, err = f.SetBit(tx, r.row, r.col, nil) + if err != nil { + t.Fatalf("failed to set bit: %v", err) + } + + _, err = existencefield.SetBit(tx, r.row, r.col, nil) + if err != nil { + t.Fatalf("failed to set bit: %v", err) + } + } + + if err = tx.Commit(); err != nil { + t.Fatalf("failed to commit tx: %v", err) + } + + err = h.processDeleteInflight() + if err != nil { + t.Fatalf("failed to delete: %v", err) + } + + tx = idx.Txf().NewTx(Txo{Write: false, Index: idx, Shard: shard}) + defer tx.Rollback() + for _, r := range rowCol { + row, err := f.Row(tx, r.row) + if err != nil { + t.Fatalf("failed to get row: %v", err) + } + existenceRow, err := existencefield.Row(tx, r.row) + if err != nil { + t.Fatalf("failed to get row: %v", err) + } + if len(row.Columns()) != 0 || len(existenceRow.Columns()) != 0 { + t.Fatalf("expected columns for fields to be empty after delete") + } + } +} diff --git a/holder_test.go b/holder_test.go index 1485f59fc..bd5edd5d7 100644 --- a/holder_test.go +++ b/holder_test.go @@ -453,7 +453,7 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) { if err != nil { t.Fatalf("creating index i: %v", err) } - _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum))) + _, err = c.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum(quantum), "0")) if err != nil { t.Fatalf("creating field f: %v", err) } diff --git a/http_handler.go b/http_handler.go index 44663ef14..3fc47cad4 100644 --- a/http_handler.go +++ b/http_handler.go @@ -429,6 +429,8 @@ func newRouter(handler *Handler) http.Handler { //router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handleGetFields, authz.Read)).Methods("GET") // Not implemented. router.HandleFunc("/index/{index}/field", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField") router.HandleFunc("/index/{index}/field/", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField") + router.HandleFunc("/index/{index}/field/{field}/view", handler.chkAuthZ(handler.handleGetView, authz.Admin)).Methods("GET") + router.HandleFunc("/index/{index}/field/{field}/view/{view}", handler.chkAuthZ(handler.handleDeleteView, authz.Admin)).Methods("DELETE").Name("DeleteView") router.HandleFunc("/index/{index}/field/{field}", handler.chkAuthZ(handler.handlePostField, authz.Write)).Methods("POST").Name("PostField") router.HandleFunc("/index/{index}/field/{field}", handler.chkAuthZ(handler.handleDeleteField, authz.Write)).Methods("DELETE").Name("DeleteField") router.HandleFunc("/index/{index}/field/{field}/import", handler.chkAuthZ(handler.handlePostImport, authz.Write)).Methods("POST").Name("PostImport") @@ -452,7 +454,6 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") // /ui endpoints are for UI use; they may change at any time. - router.HandleFunc("/ui/usage", handler.chkAuthZ(handler.handleGetUsage, authz.Read)).Methods("GET").Name("GetUsage") router.HandleFunc("/ui/transaction", handler.chkAuthZ(handler.handleGetTransactionList, authz.Read)).Methods("GET").Name("GetTransactionList") router.HandleFunc("/ui/transaction/", handler.chkAuthZ(handler.handleGetTransactionList, authz.Read)).Methods("GET").Name("GetTransactionList") router.HandleFunc("/ui/shard-distribution", handler.chkAuthZ(handler.handleGetShardDistribution, authz.Admin)).Methods("GET").Name("GetShardDistribution") @@ -466,6 +467,7 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/internal/translate/data", handler.chkAuthZ(handler.handlePostTranslateData, authz.Write)).Methods("POST").Name("PostTranslateData") // other ones + router.HandleFunc("/internal/mem-usage", handler.chkAuthZ(handler.handleGetMemUsage, authz.Read)).Methods("GET").Name("GetUsage") router.HandleFunc("/internal/fragment/block/data", handler.chkAuthN(handler.handleGetFragmentBlockData)).Methods("GET").Name("GetFragmentBlockData") router.HandleFunc("/internal/fragment/blocks", handler.chkAuthN(handler.handleGetFragmentBlocks)).Methods("GET").Name("GetFragmentBlocks") router.HandleFunc("/internal/fragment/data", handler.chkAuthN(handler.handleGetFragmentData)).Methods("GET").Name("GetFragmentData") @@ -926,7 +928,12 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { } } -// handleGetSchema handles GET /schema/details requests. +// handleGetSchema handles GET /schema/details requests. This is essentially the +// same thing as a GET /schema request, except WithViews is turned on by default. +// Previously, /schema/details returned the cardinality of each field, but this was +// removed for performance reasons. If, at some point in the future, there is a more +// performant way to get the cardinality of a field, that information would be +// included here. func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) @@ -934,7 +941,7 @@ func (h *Handler) handleGetSchemaDetails(w http.ResponseWriter, r *http.Request) } w.Header().Set("Content-Type", "application/json") - schema, err := h.api.SchemaDetails(r.Context()) + schema, err := h.api.Schema(r.Context(), true) if err != nil { h.logger.Printf("error getting detailed schema: %s", err) return @@ -987,60 +994,22 @@ func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -// handleGetUsage handles GET /ui/usage requests. -func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) { +// handleGetMemUsage handles GET /internal/mem-usage requests. +func (h *Handler) handleGetMemUsage(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } - q := r.URL.Query() - remoteStr := q.Get("remote") - var remote bool - if remoteStr == "true" { - remote = true - } - - nodeUsages, err := h.api.Usage(r.Context(), remote) + use, err := GetMemoryUsage() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) - } - - // if auth is turned on, filter results - if h.auth != nil { - g := r.Context().Value(contextKeyGroupMembership) - if g == nil { - http.Error(w, "Forbidden", http.StatusForbidden) - return - } - if !h.permissions.IsAdmin(g.([]authn.Group)) { - allowed := h.permissions.GetAuthorizedIndexList(g.([]authn.Group), authz.Read) - filteredNodeUsages := map[string]NodeUsage{} - - for nodeId, nodeUsage := range nodeUsages { - filteredIndexUsage := NodeUsage{ - Disk: DiskUsage{ - IndexUsage: map[string]IndexUsage{}, - }, - } - for index, idxUsage := range nodeUsage.Disk.IndexUsage { - // is it in auth list - for _, authd := range allowed { - if index == authd { - filteredIndexUsage.Disk.IndexUsage[index] = idxUsage - break - } - } - } - filteredNodeUsages[nodeId] = filteredIndexUsage - } - nodeUsages = filteredNodeUsages - } + return } w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(nodeUsages); err != nil { - h.logger.Errorf("write status response error: %s", err) + if err := json.NewEncoder(w).Encode(use); err != nil { + h.logger.Errorf("write mem usage response error: %s", err) } } @@ -1314,6 +1283,62 @@ func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound) } +// handleGetView handles GET /index//field//view requests. +func (h *Handler) handleGetView(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + index, err := h.api.Index(r.Context(), indexName) + + if err != nil { + http.Error(w, fmt.Sprintf("Index %s Not Found", indexName), http.StatusNotFound) + return + } else { + w.Header().Set("Content-Type", "application/json") + var viewsList []viewReponse + for _, field := range index.fields { + if field.name == fieldName { + for _, view := range field.views() { + viewsList = append(viewsList, viewReponse{Name: view.name, Type: view.fieldType, Field: view.field, Index: view.index}) + } + + if err := json.NewEncoder(w).Encode(viewsList); err != nil { + h.logger.Errorf("write response error: %s", err) + } + return + } + } + } + http.Error(w, fmt.Sprintf("Field %s Not Found", fieldName), http.StatusNotFound) +} + +type viewReponse struct { + Name string `json:"name"` + Type string `json:"type"` + Field string `json:"field"` + Index string `json:"index"` +} + +// handleDeleteIndex handles DELETE /index//field//view/ request. +func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } + + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + viewName := mux.Vars(r)["view"] + + resp := successResponse{h: h} + err := h.api.DeleteView(r.Context(), indexName, fieldName, viewName) + resp.write(w, err) +} + type postIndexRequest struct { Options IndexOptions `json:"options"` } @@ -1557,7 +1582,11 @@ func fieldOptionsToFunctionalOpts(opt fieldOptions) []FieldOption { } fos = append(fos, OptFieldTypeTimestamp(opt.Epoch.UTC(), *opt.TimeUnit)) case FieldTypeTime: - fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, opt.NoStandardView)) + if opt.Ttl != nil { + fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, *opt.Ttl, opt.NoStandardView)) + } else { + fos = append(fos, OptFieldTypeTime(*opt.TimeQuantum, "0", opt.NoStandardView)) + } case FieldTypeMutex: fos = append(fos, OptFieldTypeMutex(*opt.CacheType, *opt.CacheSize)) case FieldTypeBool: @@ -1643,14 +1672,18 @@ func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) { qcx := h.api.Txf().NewQcx() err := h.api.IngestOperations(r.Context(), qcx, indexName, r.Body) - if err == nil { - err = qcx.Finish() - if err != nil { - http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + if err != nil { + qcx.Abort() + switch e := err.(type) { + case RedirectError: + http.Redirect(w, r, e.HostPort+r.URL.Path, http.StatusPermanentRedirect) return } - } else { - qcx.Abort() + } + err = qcx.Finish() + if err != nil { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + return } resp := successResponse{h: h, Name: indexName} @@ -1679,6 +1712,7 @@ type fieldOptionSpec struct { Epoch *time.Time `json:"epoch"` Unit *string `json:"unit"` TimeQuantum *string `json:"time-quantum"` + Ttl *string `json:"ttl"` } func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { @@ -1712,159 +1746,11 @@ func fieldSpecToFieldOption(fSpec fieldSpec) fieldOptions { timeQuantumVal := TimeQuantum(*fSpec.FieldOptions.TimeQuantum) opt.TimeQuantum = &timeQuantumVal } + opt.Ttl = fSpec.FieldOptions.Ttl return opt } -// applyOneIngestSchema applies a single ingestSpec, which specifies operations on -// a single index and possibly fields. If it is successful, it returns the name -// of the index and an empty slice (if it created the index), or the name of the -// index and a slice of the fields within that index that it created. If it -// is unsuccessful, it tries to delete whatever it created. -// -// The intended idiom is that if the returned list of fields isn't empty, the index -// already existed and only those fields need to be cleaned up in the event of -// a later error, but if the list of fields is empty, the entire index was new, -// and should be cleaned up, in which case there's no need to track or delete -// the specific fields separately. -func (h *Handler) applyOneIngestSchema(ctx context.Context, schema *ingestSpec) (index *Index, returnedFields []string, err error) { - // create index - indexName := schema.IndexName - var createdFields []string - var useKeys bool - switch schema.PrimaryKeyType { - case "string": - useKeys = true - case "uint": - useKeys = false - default: - return nil, nil, fmt.Errorf("invalid primary key type %q", schema.PrimaryKeyType) - } - opts := IndexOptions{ - Keys: useKeys, - TrackExistence: true, - } - createdIndex := false - - // We check this up here because, if there's at least one field but we don't know what to do with - // it, we will necessarily fail, which means we'd delete the index anyway, so there's no point in - // trying to create it. We don't care about this if there's no fields specified. - if len(schema.Fields) > 0 { - switch schema.FieldAction { - case "create", "ensure", "require": - // do nothing - case "": - schema.FieldAction = schema.IndexAction - default: - return nil, nil, fmt.Errorf("invalid field-action %q, expecting create/ensure/require", schema.FieldAction) - } - } - - switch schema.IndexAction { - case "ensure", "require": - index, err = h.api.Index(ctx, indexName) - if err != nil { - if _, ok := err.(NotFoundError); !ok { - return nil, nil, fmt.Errorf("checking for existing index %q: %w", indexName, err) - } else { - err = nil - } - } - if index != nil { - existingOpts := index.Options() - if existingOpts != opts { - return nil, nil, fmt.Errorf("index %q options mismatch: schema %#v, existing %#v", indexName, opts, existingOpts) - } - break - } - if schema.IndexAction == "require" { - return nil, nil, fmt.Errorf("index %q does not exist", indexName) - } - fallthrough - case "create": - index, err = h.api.CreateIndex(ctx, indexName, opts) - if err != nil { - return nil, nil, err - } - createdIndex = true - default: - return nil, nil, fmt.Errorf("invalid index-action %q, need create/ensure/require", schema.IndexAction) - } - - // Now we might have an index, so we need our cleanup code. - defer func() { - if err == nil { - return - } - if createdIndex { - err := h.api.DeleteIndex(ctx, indexName) - if err != nil { - h.logger.Printf("trying to undo failed index %q creation: %v", indexName, err) - } - return - } - for _, field := range createdFields { - err := h.api.DeleteField(ctx, indexName, field) - if err != nil { - h.logger.Printf("trying to undo failed field %q creation in index %q: %v", field, indexName, err) - } - } - }() - - // create all the fields specified in the index - for _, fSpec := range schema.Fields { - fieldName := fSpec.FieldName - opt := fieldSpecToFieldOption(fSpec) - err = opt.validate() - if err != nil { - return nil, nil, err - } - switch schema.FieldAction { - case "ensure", "require": - field, schemaErr := h.api.Field(ctx, indexName, fieldName) - if schemaErr != nil { - // NotFoundError is fine - if _, ok := schemaErr.(NotFoundError); !ok { - return nil, nil, fmt.Errorf("checking for existing field %q in %q: %w", fieldName, indexName, err) - } - } - if field != nil { - existing := field.Options() - if opt.Type != existing.Type { - return nil, nil, fmt.Errorf("existing field %q is %q, not %q", fieldName, existing.Type, opt.Type) - } - if ((opt.Keys != nil) && *opt.Keys) != existing.Keys { - if existing.Keys { - return nil, nil, fmt.Errorf("existing field %q in %q uses keys", fieldName, indexName) - } else { - return nil, nil, fmt.Errorf("existing field %q in %q doesn't use keys", fieldName, indexName) - } - } - // TODO: verify compatibility of other field opts, this is sorta hard - break - } - if schema.FieldAction == "require" { - return nil, nil, fmt.Errorf("field %q does not exist in %q", fieldName, indexName) - } - fallthrough - case "create": - fos := fieldOptionsToFunctionalOpts(opt) - _, err = h.api.CreateField(ctx, indexName, fieldName, fos...) - if err != nil { - return nil, nil, fmt.Errorf("creating field %q in %q: %v", fieldName, indexName, err) - } - createdFields = append(createdFields, fieldName) - } - } - // we don't report the fields back, so we can distinguish "created index" - // from "created fields within index" - if createdIndex { - createdFields = nil - } - - return index, createdFields, nil -} - func (h *Handler) handleIngestSchema(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) @@ -1907,12 +1793,18 @@ func (h *Handler) handleIngestSchema(w http.ResponseWriter, r *http.Request) { resp.write(w, err) return } - index, fields, err := h.applyOneIngestSchema(r.Context(), &schema) + index, fields, err := h.api.ApplyOneIngestSchema(r.Context(), &schema) if err != nil { - // if a previous schema created things, clean them up... - schemaErr = err - resp.write(w, err) - return + switch e := err.(type) { + case RedirectError: + http.Redirect(w, r, e.HostPort+r.URL.Path, http.StatusPermanentRedirect) + return + default: + // if a previous schema created things, clean them up... + schemaErr = err + resp.write(w, err) + return + } } // we only have one slot to report these, sorry. resp.Name = index.Name() @@ -1951,6 +1843,7 @@ type fieldOptions struct { Keys *bool `json:"keys,omitempty"` NoStandardView bool `json:"noStandardView,omitempty"` ForeignIndex *string `json:"foreignIndex,omitempty"` + Ttl *string `json:"ttl,omitempty"` } func (o *fieldOptions) validate() error { @@ -1978,6 +1871,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("max does not apply to field type set")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type set")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type set")) } case FieldTypeInt: if o.CacheType != nil { @@ -1986,6 +1881,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("cacheSize does not apply to field type int")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type int")) } case FieldTypeDecimal: if o.Scale == nil { @@ -1996,6 +1893,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("cacheSize does not apply to field type int")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type int")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type int")) } else if o.ForeignIndex != nil && o.Type == FieldTypeDecimal { return NewBadRequestError(errors.New("decimal field cannot be a foreign key")) } @@ -2010,6 +1909,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("cacheSize does not apply to field type timestamp")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type timestamp")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type timestamp")) } else if o.ForeignIndex != nil { return NewBadRequestError(errors.New("timestamp field cannot be a foreign key")) } @@ -2038,6 +1939,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("max does not apply to field type mutex")) } else if o.TimeQuantum != nil { return NewBadRequestError(errors.New("timeQuantum does not apply to field type mutex")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type mutex")) } case FieldTypeBool: if o.CacheType != nil { @@ -2052,6 +1955,8 @@ func (o *fieldOptions) validate() error { return NewBadRequestError(errors.New("timeQuantum does not apply to field type bool")) } else if o.Keys != nil { return NewBadRequestError(errors.New("keys does not apply to field type bool")) + } else if o.Ttl != nil { + return NewBadRequestError(errors.New("ttl does not apply to field type bool")) } else if o.ForeignIndex != nil { return NewBadRequestError(errors.New("bool field cannot be a foreign key")) } diff --git a/http_handler_internal_test.go b/http_handler_internal_test.go index 455a40c64..30bf3a782 100644 --- a/http_handler_internal_test.go +++ b/http_handler_internal_test.go @@ -131,6 +131,7 @@ func TestFieldOptionValidation(t *testing.T) { {json: `{"options": {"type": "set", "min": 0}}`, err: "min does not apply to field type set"}, {json: `{"options": {"type": "set", "max": 100}}`, err: "max does not apply to field type set"}, {json: `{"options": {"type": "set", "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type set"}, + {json: `{"options": {"type": "set", "ttl": "1h"}}`, err: "ttl does not apply to field type set"}, // FieldType: Int {json: `{"options": {"type": "int"}}`, err: "min is required for field type int"}, @@ -143,6 +144,7 @@ func TestFieldOptionValidation(t *testing.T) { {json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheType": "ranked"}}`, err: "cacheType does not apply to field type int"}, {json: `{"options": {"type": "int", "min": 0, "max": 1000, "cacheSize": 1000}}`, err: "cacheSize does not apply to field type int"}, {json: `{"options": {"type": "int", "min": 0, "max": 1000, "timeQuantum": "YMD"}}`, err: "timeQuantum does not apply to field type int"}, + {json: `{"options": {"type": "int", "min": 0, "max": 1000, "ttl": "1h"}}`, err: "ttl does not apply to field type int"}, // FieldType: Time {json: `{"options": {"type": "time"}}`, err: "timeQuantum is required for field type time"}, @@ -771,3 +773,19 @@ func NewTestAuth(t *testing.T) *authn.Auth { } return a } + +func TestHandleGetMemUsage(t *testing.T) { + h := Handler{ + logger: logger.NewStandardLogger(os.Stdout), + queryLogger: logger.NewStandardLogger(os.Stdout), + } + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/whatever", nil) + + h.handleGetMemUsage(w, r) + + resp := w.Result() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected %v, got %v", http.StatusOK, resp.StatusCode) + } +} diff --git a/http_handler_test.go b/http_handler_test.go index a692a8435..2a213642b 100644 --- a/http_handler_test.go +++ b/http_handler_test.go @@ -6,6 +6,8 @@ import ( "fmt" "net" gohttp "net/http" + "reflect" + "sort" "strings" "testing" @@ -170,6 +172,166 @@ func TestIngestSchemaHandler(t *testing.T) { } } +func TestPostFieldWithTtl(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + + schema := ` + { + "index-name": "example", + "primary-key-type": "string", + "index-action": "create", + "fields":[] + } + ` + m := c.GetPrimary() + schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) + resp := test.Do(t, "POST", schemaURL, string(schema)) + if resp.StatusCode != gohttp.StatusOK { + t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + + postFieldTtlUrl := fmt.Sprintf("%s/index/example/field/with_ttl", m.URL()) + + // Create new field with ttl but in invalid format + fieldOptionInvalidTtl := ` + { "options": {"timeQuantum":"YMDH","type":"time","ttl":"24hour" }} + ` + respField := test.Do(t, "POST", postFieldTtlUrl, string(fieldOptionInvalidTtl)) + if (respField.StatusCode != gohttp.StatusBadRequest) && + (respField.Body != "applying option: cannot parse ttl: 24hour") { + t.Errorf("expected ttl parse error, got status: %d, body=%s", respField.StatusCode, respField.Body) + } + + // Create new field with ttl in invalid format + fieldOptionValidTtl := ` + { "options": {"timeQuantum":"YMDH","type":"time","ttl":"24h" }} + ` + respField = test.Do(t, "POST", postFieldTtlUrl, string(fieldOptionValidTtl)) + if resp.StatusCode != gohttp.StatusOK { + t.Errorf("creating field with ttl, got status: %d, body=%s", respField.StatusCode, respField.Body) + } + + // Create new field without ttl + postFieldNoTtlUrl := fmt.Sprintf("%s/index/example/field/no_ttl", m.URL()) + fieldOptionNoTtl := ` + { "options": {"timeQuantum":"YMDH","type":"time" }} + ` + respField = test.Do(t, "POST", postFieldNoTtlUrl, string(fieldOptionNoTtl)) + if resp.StatusCode != gohttp.StatusOK { + t.Errorf("creating field without ttl, status: %d, body=%s", respField.StatusCode, respField.Body) + } +} + +func TestGetViewAndDelete(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + + schema := ` + { + "index-name": "example", + "primary-key-type": "string", + "index-action": "create", + "fields": [ + { + "field-name": "test_view", + "field-type": "time", + "field-options": { + "time-quantum": "YMDH" + } + } + ] + } + ` + m := c.GetPrimary() + schemaURL := fmt.Sprintf("%s/internal/schema", m.URL()) + resp := test.Do(t, "POST", schemaURL, string(schema)) + if resp.StatusCode != gohttp.StatusOK { + t.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body) + } + + // Send sample data + postQueryUrl := fmt.Sprintf("%s/index/example/query", m.URL()) + queryOption := ` + Set(1,test_view=1,2001-02-03T04:05) + ` + respQuery := test.Do(t, "POST", postQueryUrl, string(queryOption)) + if respQuery.StatusCode != gohttp.StatusOK { + t.Errorf("posting query, status: %d, body=%s", respQuery.StatusCode, respQuery.Body) + } + + // The above sample data should create these views: + expectedViewNames := []string{ + "standard", + "standard_2001", + "standard_200102", + "standard_20010203", + "standard_2001020304", + } + + // Call view to get data + viewUrl := fmt.Sprintf("%s/index/example/field/test_view/view", m.URL()) + respView := test.Do(t, "GET", viewUrl, "") + if respView.StatusCode != gohttp.StatusOK { + t.Errorf("view handler, status: %d, body=%s", respView.StatusCode, respView.Body) + } + + type viewReponse struct { + Name string `json:"name"` + Type string `json:"type"` + Field string `json:"field"` + Index string `json:"index"` + } + + var parsedViews []viewReponse + if err := json.Unmarshal([]byte(respView.Body), &parsedViews); err != nil { + t.Errorf("parsing view, err: %s", err) + } + + // check if data from view matches with expectedViewNames + parseViewNames := []string{} + for _, view := range parsedViews { + parseViewNames = append(parseViewNames, view.Name) + } + sort.Strings(parseViewNames) + + if !reflect.DeepEqual(expectedViewNames, parseViewNames) { + t.Fatalf("expected %v, but got %v", expectedViewNames, parseViewNames) + } + + // call delete on view standard_2001020304 + deleteViewUrl := fmt.Sprintf("%s/index/example/field/test_view/view/standard_2001020304", m.URL()) + respDelete := test.Do(t, "DELETE", deleteViewUrl, "") + if respDelete.StatusCode != gohttp.StatusOK { + t.Errorf("delete handler, status: %d, body=%s", respDelete.StatusCode, respDelete.Body) + } + + // remove view that was deleted (standard_2001020304) from expectedViewNames + expectedViewNames = expectedViewNames[:len(expectedViewNames)-1] + + // call view again + viewUrl = fmt.Sprintf("%s/index/example/field/test_view/view", m.URL()) + respView = test.Do(t, "GET", viewUrl, "") + if respView.StatusCode != gohttp.StatusOK { + t.Errorf("view handler after delete, status: %d, body=%s", respView.StatusCode, respView.Body) + } + + if err := json.Unmarshal([]byte(respView.Body), &parsedViews); err != nil { + t.Errorf("parsing view, err: %s", err) + } + + // check if data from view matches with expectedViewNames + parseViewNames = []string{} + for _, view := range parsedViews { + parseViewNames = append(parseViewNames, view.Name) + } + sort.Strings(parseViewNames) + + if !reflect.DeepEqual(expectedViewNames, parseViewNames) { + t.Fatalf("after delete, expected %v, but got %v", expectedViewNames, parseViewNames) + } +} + func TestTranslationHandlers(t *testing.T) { // reusable data for the tests nameBytes, err := json.Marshal([]string{"a", "b", "c"}) diff --git a/index.go b/index.go index 0e900cb05..12e0aee68 100644 --- a/index.go +++ b/index.go @@ -272,16 +272,14 @@ func (i *Index) openFields(idx *disco.Index) error { } fileLoop: for fname, fld := range idx.Fields { + lfname := fname select { case <-ctx.Done(): break fileLoop default: - var cfm *CreateFieldMessage = &CreateFieldMessage{} - var err error - // Decode the CreateFieldMessage from the schema data in order to // get its metadata. - cfm, err = decodeCreateFieldMessage(i.holder.serializer, fld.Data) + cfm, err := decodeCreateFieldMessage(i.holder.serializer, fld.Data) if err != nil { return errors.Wrap(err, "decoding create field message") } @@ -291,9 +289,9 @@ fileLoop: defer func() { <-indexQueue }() - i.holder.Logger.Debugf("open field: %s", fname) + i.holder.Logger.Debugf("open field: %s", lfname) - _, err := i.openField(&mu, cfm, fname) + _, err := i.openField(&mu, cfm, lfname) if err != nil { return errors.Wrap(err, "opening field") } @@ -505,12 +503,21 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) { return nil, errors.Wrap(err, "validating name") } - i.mu.Lock() - defer i.mu.Unlock() + // Grab lock, check for field existing, release lock. We don't want + // to stay holding the lock, but we might care about the ErrFieldExists + // part of this. + err = func() error { + i.mu.Lock() + defer i.mu.Unlock() - // Ensure field doesn't already exist. - if i.fields[name] != nil { - return nil, newConflictError(ErrFieldExists) + // Ensure field doesn't already exist. + if i.fields[name] != nil { + return newConflictError(ErrFieldExists) + } + return nil + }() + if err != nil { + return nil, err } // Apply and validate functional options. @@ -526,37 +533,26 @@ func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error) { Meta: fo, } - // Create the field in etcd as the system of record. + // Create the field in etcd as the system of record. We do this without + // the lock held because it can take an arbitrary amount of time... if err := i.persistField(context.Background(), cfm); err != nil { return nil, errors.Wrap(err, "persisting field") } - return i.createField(cfm, false) -} - -// CreateFieldAndBroadcast creates a field locally, then broadcasts the -// creation to other nodes so they can create locally as well. An error is -// returned if the field already exists. -func (i *Index) CreateFieldAndBroadcast(cfm *CreateFieldMessage) (*Field, error) { - err := ValidateName(cfm.Field) - if err != nil { - return nil, errors.Wrap(err, "validating name") - } - + // This is identical to the previous check, because we could get super + // unlucky and have the persist-field thing happen, and somehow the field + // gets created, before we get to run again, and the specific nature of + // the error can matter to the backend. i.mu.Lock() defer i.mu.Unlock() // Ensure field doesn't already exist. - if i.fields[cfm.Field] != nil { + if i.fields[name] != nil { return nil, newConflictError(ErrFieldExists) } - // Create the field in etcd as the system of record. - if err := i.persistField(context.Background(), cfm); err != nil { - return nil, errors.Wrap(err, "persisting field") - } - - return i.createField(cfm, true) + // Actually do the internal bookkeeping. + return i.createField(cfm) } // CreateFieldIfNotExists creates a field with the given options if it doesn't exist. @@ -596,7 +592,7 @@ func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field return nil, errors.Wrap(err, "persisting field") } - return i.createField(cfm, false) + return i.createField(cfm) } // CreateFieldIfNotExistsWithOptions is a method which I created because I @@ -634,7 +630,7 @@ func (i *Index) CreateFieldIfNotExistsWithOptions(name string, opt *FieldOptions return nil, errors.Wrap(err, "persisting field") } - return i.createField(cfm, false) + return i.createField(cfm) } // persistField stores the field information in etcd. @@ -669,14 +665,14 @@ func (i *Index) createFieldIfNotExists(cfm *CreateFieldMessage) (*Field, error) return f, nil } - return i.createField(cfm, false) + return i.createField(cfm) } -// createField, in addition to creating a new Field, calls Field.Open which -// potentially aquires a lock on Index. So until/unless we refactor the -// Index.createField() function call path, we cannot call Index.createField -// while holding an Index lock. -func (i *Index) createField(cfm *CreateFieldMessage, broadcast bool) (*Field, error) { +// createField does the internal field creation logic, creating the in-memory +// data structure, and kicking translation sync if appropriate. It does not +// notify other nodes; that's done from the API's initial CreateField call +// now. +func (i *Index) createField(cfm *CreateFieldMessage) (*Field, error) { opt := cfm.Meta if opt == nil { opt = &FieldOptions{} @@ -713,13 +709,6 @@ func (i *Index) createField(cfm *CreateFieldMessage, broadcast bool) (*Field, er // enable Txf to find the index in field_test.go TestField_SetValue f.idx = i - if broadcast { - // Send the create field message to all nodes. - if err := i.holder.sendOrSpool(cfm); err != nil { - return nil, errors.Wrap(err, "sending CreateField message") - } - } - // Kick off the field's translation sync process. if err := i.translationSyncer.Reset(); err != nil { return nil, errors.Wrap(err, "resetting translation syncer") diff --git a/index_test.go b/index_test.go index 67d69ecbe..7d63f8639 100644 --- a/index_test.go +++ b/index_test.go @@ -55,7 +55,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with explicit quantum. - f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")) if err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { @@ -71,7 +71,7 @@ func TestIndex_CreateField(t *testing.T) { defer index.Close() // Create field with explicit quantum with no standard view - f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), true)) + f, err := index.CreateField("f", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0", true)) if err != nil { t.Fatal(err) } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { diff --git a/install/featurebase.conf b/install/featurebase.conf index a8894e8f9..6068046b0 100644 --- a/install/featurebase.conf +++ b/install/featurebase.conf @@ -244,28 +244,6 @@ log-path = "/var/log/molecula/featurebase.log" # enable-client-verification = true - -# ============================================================================== -# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is -# calculated periodically in the background and accessed by the UI/usage -# endpoint. Since this disk scan can take a long and unpredictable amount of -# time, its timing behavior is specified in a relative, rather than absolute -# sense. That is, the duty cycle sets the percentage of time that is spent -# recalculating this cache. This setting affects the results received from -# the "/ui/usage" http endpoint, as well as all data file and memory usage -# values and graphs on the webui "tables" page - -# Special considerations: -# * If disk usage can be calculated quickly (less than 5 seconds), fresh -# results will be calculated when accessed -# * When disk usage takes longer to calculate, there is a minimum of one -# hour wait between cache recalculations -# Setting this value to 0 will completely disable the calculation of disk usage -# -# usage-duty-cycle = 20 - - - # ============================================================================== # Use [metric] stanza to define attributes for monitoring. # [metric] diff --git a/internal/clustertests/Dockerfile-fakeIDP b/internal/clustertests/Dockerfile-fakeIDP index b53d4d2c2..b46556b80 100644 --- a/internal/clustertests/Dockerfile-fakeIDP +++ b/internal/clustertests/Dockerfile-fakeIDP @@ -1,4 +1,4 @@ -FROM golang:latest +FROM golang:1.16 WORKDIR / COPY fakeidp ./ diff --git a/internal/clustertests/testdata/featurebase.conf b/internal/clustertests/testdata/featurebase.conf index eb587fbcb..e71ac5a9c 100644 --- a/internal/clustertests/testdata/featurebase.conf +++ b/internal/clustertests/testdata/featurebase.conf @@ -244,28 +244,6 @@ # enable-client-verification = true - -# ============================================================================== -# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is -# calculated periodically in the background and accessed by the UI/usage -# endpoint. Since this disk scan can take a long and unpredictable amount of -# time, its timing behavior is specified in a relative, rather than absolute -# sense. That is, the duty cycle sets the percentage of time that is spent -# recalculating this cache. This setting affects the results received from -# the "/ui/usage" http endpoint, as well as all data file and memory usage -# values and graphs on the webui "tables" page - -# Special considerations: -# * If disk usage can be calculated quickly (less than 5 seconds), fresh -# results will be calculated when accessed -# * When disk usage takes longer to calculate, there is a minimum of one -# hour wait between cache recalculations -# Setting this value to 0 will completely disable the calculation of disk usage -# -# usage-duty-cycle = 20 - - - # ============================================================================== # Use [metric] stanza to define attributes for monitoring. # [metric] diff --git a/internal_client.go b/internal_client.go index fa1bc6653..1bff1e90a 100644 --- a/internal_client.go +++ b/internal_client.go @@ -1094,6 +1094,8 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel fieldOpt.Max = &opt.Max case FieldTypeTime: fieldOpt.TimeQuantum = &opt.TimeQuantum + ttlString := opt.Ttl.String() + fieldOpt.Ttl = &ttlString case FieldTypeBool: // pass case FieldTypeDecimal: @@ -1383,38 +1385,6 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in return tkresp.Keys, nil } -// GetNodeUsage retrieves the size-on-disk information for the specified node. -func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[string]NodeUsage, error) { - u := uri.Path("/ui/usage?remote=true") - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, errors.Wrap(err, "creating request") - } - - req.Header.Set("Accept", "application/json") - req.Header.Set("User-Agent", "pilosa/"+Version) - req = AddAuthToken(ctx, req) - - // Execute request against the host. - resp, err := c.executeRequest(req.WithContext(ctx)) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - // Read body and unmarshal response. - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, "reading") - } - - nodeUsages := make(map[string]NodeUsage) // map of size 1 - if err := json.Unmarshal(body, &nodeUsages); err != nil { - return nil, fmt.Errorf("unmarshal response: %s", err) - } - return nodeUsages, nil -} - // GetPastQueries retrieves the query history log for the specified node. func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]PastQueryStatus, error) { u := uri.Path("/query-history?remote=true") diff --git a/internal_client_test.go b/internal_client_test.go index 8d1531137..204a85cae 100644 --- a/internal_client_test.go +++ b/internal_client_test.go @@ -1175,6 +1175,46 @@ func TestClient_FragmentBlocks(t *testing.T) { } } +func TestClient_CreateTimeField(t *testing.T) { + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + cmd := cluster.GetNode(0) + + c := MustNewClient(cmd.URL(), pilosa.GetHTTPClient(nil)) + + index := "cdf" + err := c.CreateIndex(context.Background(), index, pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + + field := "field" + err = c.CreateFieldWithOptions(context.Background(), index, field, pilosa.FieldOptions{Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}) + if err != nil { + t.Fatalf("creating field: %v", err) + } + fld, err := cmd.API.Field(context.Background(), index, field) + if err != nil { + t.Fatalf("getting field: %v", err) + } + if fld.Ttl() != 0 { + t.Fatalf("expected Ttl to be 0, got: %+v", fld.Options().Ttl.String()) + } + + fieldTtl := "field_ttl" + err = c.CreateFieldWithOptions(context.Background(), index, fieldTtl, pilosa.FieldOptions{Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH", Ttl: time.Hour}) + if err != nil { + t.Fatalf("creating field: %v", err) + } + fldTtl, err := cmd.API.Field(context.Background(), index, fieldTtl) + if err != nil { + t.Fatalf("getting field: %v", err) + } + if fldTtl.Ttl() != time.Hour { + t.Fatalf("expected Ttl 1 hour, got: %+v", fldTtl.Ttl().String()) + } +} + func TestClient_CreateDecimalField(t *testing.T) { cluster := test.MustRunCluster(t, 1) defer cluster.Close() diff --git a/lattice/src/App/Home/ClusterHealth/ClusterHealth.tsx b/lattice/src/App/Home/ClusterHealth/ClusterHealth.tsx index 2809b2fd2..5dc836ad7 100644 --- a/lattice/src/App/Home/ClusterHealth/ClusterHealth.tsx +++ b/lattice/src/App/Home/ClusterHealth/ClusterHealth.tsx @@ -19,14 +19,12 @@ export const ClusterHealth: FC = () => { const [cluster, setCluster] = useState(); const [metrics, setMetrics] = useState(); const [info, setInfo] = useState(); - const [clusterData, setClusterData] = useState(); const [expanded, setExpanded] = useState([]); const [showMetrics, setShowMetrics] = useState(); const allExpanded = cluster && expanded.length === cluster.nodes.length; useEffectOnce(() => { getClusterHealth(); - getClusterData(); }); const refreshMetrics = useCallback(() => { @@ -38,15 +36,11 @@ export const ClusterHealth: FC = () => { useEffect(() => { const interval = setInterval(() => { - if (!clusterData) { - getClusterData(); - } - getClusterHealth(); refreshMetrics(); }, 15000); return () => clearInterval(interval); - }, [refreshMetrics, cluster, clusterData]); + }, [refreshMetrics, cluster]); const getClusterHealth = () => { pilosa.get @@ -76,13 +70,6 @@ export const ClusterHealth: FC = () => { .catch(() => setMetrics(undefined)); }; - const getClusterData = () => { - pilosa.get - .usage() - .then((res) => setClusterData(res.data)) - .catch(() => setClusterData(undefined)); - }; - const toggleAccordion = (nodeId: string) => { const isExpanded = expanded.includes(nodeId); if (isExpanded) { @@ -140,7 +127,6 @@ export const ClusterHealth: FC = () => { key={node.id} node={node} info={info} - usage={clusterData ? clusterData[node.id] : undefined} expanded={expanded.includes(node.id)} onToggle={() => toggleAccordion(node.id)} onMetricClick={() => setShowMetrics(node)} diff --git a/lattice/src/App/Home/ClusterHealth/Node/Node.tsx b/lattice/src/App/Home/ClusterHealth/Node/Node.tsx index d9a8a5cf3..8d934fac4 100644 --- a/lattice/src/App/Home/ClusterHealth/Node/Node.tsx +++ b/lattice/src/App/Home/ClusterHealth/Node/Node.tsx @@ -1,4 +1,4 @@ -import React, { FC, Fragment, useState } from 'react'; +import React, { FC, useState } from 'react'; import Button from '@material-ui/core/Button'; import copy from 'copy-to-clipboard'; import EqualizerIcon from '@material-ui/icons/EqualizerSharp'; @@ -11,7 +11,6 @@ import Find from 'lodash/find'; import IconButton from '@material-ui/core/IconButton'; import InfoIcon from '@material-ui/icons/Info'; import Tooltip from '@material-ui/core/Tooltip'; -import Typography from '@material-ui/core/Typography'; import { formatBytes } from 'shared/utils/formatBytes'; import { nodeInfo } from './nodeInfo'; import { NODE_STATE } from './nodeStatus'; @@ -21,7 +20,6 @@ import css from './Node.module.scss'; type NodeType = { node: any; info: any; - usage: any; expanded: boolean; onToggle: () => void; onMetricClick: () => void; @@ -30,24 +28,13 @@ type NodeType = { export const Node: FC = ({ node, info, - usage, expanded, onToggle, - onMetricClick + onMetricClick, }) => { const [copyHost, setCopyHost] = useState('Copy Host'); const [copyID, setCopyID] = useState('Click to Copy'); const { id, isPrimary, state } = node; - const diskTotalInUse = usage?.diskUsage?.totalInUse; - const diskCapacity = usage?.diskUsage?.capacity; - const diskUsagePercentage = diskCapacity - ? (diskTotalInUse / diskCapacity) * 100 - : undefined; - const memoryTotalInUse = usage?.memoryUsage?.totalInUse; - const memoryCapacity = usage?.memoryUsage?.capacity; - const memoryUsagePercentage = memoryCapacity - ? (memoryTotalInUse / memoryCapacity) * 100 - : undefined; const keys = Object.keys(info); const onCopyHostClick = () => { @@ -103,154 +90,6 @@ export const Node: FC = ({ -
-
-
Disk Usage:
-
- {usage ? ( - - - {formatBytes(diskTotalInUse)} - {diskCapacity - ? ` used out of ${formatBytes(diskCapacity)}` - : null} - -
- {diskUsagePercentage ? ( - - {diskUsagePercentage < 1 - ? '< 1' - : diskUsagePercentage.toLocaleString( - undefined, - { maximumFractionDigits: 1 } - )} - % used - - } - placement="top" - arrow - > -
- - ) : ( - - - {formatBytes(diskTotalInUse)} used - - } - placement="top" - arrow - > -
- - - Node disk capacity unknown - - - )} -
-
- ) : ( - - Calculating... - - )} -
-
-
-
Memory Usage:
-
- {usage ? ( - - - {formatBytes(memoryTotalInUse)} - {memoryCapacity - ? ` used out of ${formatBytes(memoryCapacity)}` - : null} - -
- {memoryUsagePercentage ? ( - - {memoryUsagePercentage < 1 - ? '< 1' - : memoryUsagePercentage.toLocaleString( - undefined, - { maximumFractionDigits: 1 } - )} - % used - - } - placement="top" - arrow - > -
- - ) : ( - - - {formatBytes(memoryTotalInUse)} used - - } - placement="top" - arrow - > -
- - - Node memory capacity unknown - - - )} -
-
- ) : ( - - Calculating... - - )} -
-
-
{keys.map((key) => { const showNode = Find(nodeInfo, (node) => node.name === key); diff --git a/lattice/src/App/MoleculaTables/MoleculaTable/MoleculaTable.tsx b/lattice/src/App/MoleculaTables/MoleculaTable/MoleculaTable.tsx index 72faaa4c1..6c09fdf27 100644 --- a/lattice/src/App/MoleculaTables/MoleculaTable/MoleculaTable.tsx +++ b/lattice/src/App/MoleculaTables/MoleculaTable/MoleculaTable.tsx @@ -4,75 +4,38 @@ import Breadcrumbs from '@material-ui/core/Breadcrumbs'; import classNames from 'classnames'; import Fuse from 'fuse.js'; import Highlighter from 'react-highlight-words'; -import isEmpty from 'lodash/isEmpty'; import Link from '@material-ui/core/Link'; import map from 'lodash/map'; import moment from 'moment'; import OrderBy from 'lodash/orderBy'; -import Reduce from 'lodash/reduce'; import Table from '@material-ui/core/Table'; import TableBody from '@material-ui/core/TableBody'; import TableCell from '@material-ui/core/TableCell'; import TableHead from '@material-ui/core/TableHead'; import TableRow from '@material-ui/core/TableRow'; import TextField from '@material-ui/core/TextField'; -import Tooltip from '@material-ui/core/Tooltip'; import Typography from '@material-ui/core/Typography'; import { Block } from 'shared/Block'; import { Pager } from 'shared/Pager'; -import { UsageBreakdown } from '../UsageBreakdown'; import css from './MoleculaTable.module.scss'; type MoleculaTableProps = { table: any; - dataDistribution: any; lastUpdated: string; }; export const MoleculaTable: FC = ({ table, - dataDistribution, - lastUpdated + lastUpdated, }) => { const [page, setPage] = useState(1); const [resultsPerPage, setResultsPerPage] = useState(10); const sliceStart = (page - 1) * resultsPerPage; const [searchText, setSearchText] = useState(''); const [filteredFields, setFiltereedFields] = useState(table.fields); - const [fieldsData, setFieldsData] = useState<{}>({}); - const [maxFieldSize, setMaxFieldSize] = useState(0); + const [fieldsData] = useState<{}>({}); const [sort, setSort] = useState('total'); const [sortDir, setSortDir] = useState<'asc' | 'desc'>('desc'); - const lastUpdatedMoment = lastUpdated ? moment(lastUpdated).utc() : undefined; - - useEffect(() => { - if (dataDistribution && !dataDistribution.uncached) { - const aggregatedFieldsData = Reduce( - dataDistribution.fields, - (result, value) => { - let newResult = {}; - const keys = Object.keys(value); - keys.forEach( - (key) => - (newResult[key] = { - total: result[key].total + value[key].total, - fragments: result[key].fragments + value[key].fragments, - keys: result[key].keys + value[key].keys, - metadata: result[key].metadata + value[key].metadata - }) - ); - return newResult; - } - ); - - const sorted = OrderBy(aggregatedFieldsData, ['total'], ['desc']); - if (sorted.length > 0) { - setMaxFieldSize(sorted[0].total); - } - - setFieldsData(aggregatedFieldsData); - } - }, [dataDistribution]); useEffect(() => { if (searchText.length > 1) { @@ -80,7 +43,7 @@ export const MoleculaTable: FC = ({ keys: ['name'], minMatchCharLength: 2, ignoreLocation: true, - threshold: 0 + threshold: 0, }); const result = fuse.search(searchText); @@ -131,46 +94,6 @@ export const MoleculaTable: FC = ({ {table.name} - {lastUpdatedMoment ? ( -
- {dataDistribution && dataDistribution.uncached ? ( - - Disk usage will be calculated at the next{` `} - - Disk and memory information shown here are read from a - cache, the behavior of which can be controlled with the{` `} - - --usage-duty-cycle - {' '} - command line flag. - - } - placement="top" - arrow - > - cache refresh - - . - - ) : ( - - Disk usage last updated{' '} - - - {lastUpdatedMoment.fromNow()} - - - . - - )} -
- ) : null}
@@ -180,9 +103,6 @@ export const MoleculaTable: FC = ({
-
- -
@@ -211,36 +131,20 @@ export const MoleculaTable: FC = ({ onSortClick('name')} > Name{' '} Type - Cardinality Options - - onSortClick('total')} - > - Disk Usage{' '} - - - @@ -266,9 +170,6 @@ export const MoleculaTable: FC = ({ {type} {showKeys ? (keys ? '(keys)' : '(ID)') : null} - - {cardinality ? cardinality.toLocaleString() : '-'} -
{map(rest, (value, key) => { @@ -295,22 +196,6 @@ export const MoleculaTable: FC = ({ })}
- - - ); })} diff --git a/lattice/src/App/MoleculaTables/MoleculaTables.tsx b/lattice/src/App/MoleculaTables/MoleculaTables.tsx index cae1590fb..083c095d6 100644 --- a/lattice/src/App/MoleculaTables/MoleculaTables.tsx +++ b/lattice/src/App/MoleculaTables/MoleculaTables.tsx @@ -8,42 +8,29 @@ import Tooltip from '@material-ui/core/Tooltip'; import Typography from '@material-ui/core/Typography'; import { Block } from 'shared/Block'; import { SortBy } from 'shared/SortBy'; -import { UsageBreakdown } from './UsageBreakdown'; import { useHistory } from 'react-router-dom'; import css from './MoleculaTables.module.scss'; type MoleculaTablesProps = { tables: any; - dataDistribution: any; lastUpdated: string; maxSize: number; }; export const MoleculaTables: FC = ({ tables, - dataDistribution, lastUpdated, - maxSize + maxSize, }) => { const history = useHistory(); const [sortedTables, setSortedTables] = useState([]); const lastUpdatedMoment = lastUpdated ? moment(lastUpdated).utc() : undefined; useEffect(() => { - if (tables && dataDistribution) { - let aggregatedData: any[] = []; - tables.forEach((i) => - aggregatedData.push({ - ...dataDistribution[i.name], - ...i - }) - ); - - setSortedTables(aggregatedData); - } else if (tables) { + if (tables) { setSortedTables(tables); } - }, [tables, dataDistribution]); + }, [tables]); const handleSortChange = (value: any) => { const sortDirection = value === 'name' ? 'asc' : 'desc'; @@ -96,7 +83,7 @@ export const MoleculaTables: FC = ({ { label: 'Index Keys Size', value: 'indexKeys' }, { label: 'Fragment Size', value: 'fragments' }, { label: 'Field Keys Size', value: 'fieldKeysTotal' }, - { label: 'Metadata Size', value: 'metadata' } + { label: 'Metadata Size', value: 'metadata' }, ]} defaultValue="name" onChange={handleSortChange} @@ -111,22 +98,6 @@ export const MoleculaTables: FC = ({
{name}
-
- -
keys diff --git a/lattice/src/App/MoleculaTables/MoleculaTablesContainer.tsx b/lattice/src/App/MoleculaTables/MoleculaTablesContainer.tsx index 0e88eddfd..c84a84243 100644 --- a/lattice/src/App/MoleculaTables/MoleculaTablesContainer.tsx +++ b/lattice/src/App/MoleculaTables/MoleculaTablesContainer.tsx @@ -1,5 +1,4 @@ import React, { useEffect, useState } from 'react'; -import OrderBy from 'lodash/orderBy'; import { MoleculaTable } from './MoleculaTable'; import { MoleculaTables } from './MoleculaTables'; import { pilosa } from 'services/eventServices'; @@ -12,9 +11,8 @@ export const MoleculaTablesContainer = () => { const history = useHistory(); const [tables, setTables] = useState(); const [selectedTable, setSelectedTable] = useState(); - const [dataDistribution, setDataDistribution] = useState(); - const [maxSize, setMaxSize] = useState(0); - const [lastUpdated, setLastUpdated] = useState(''); + const [maxSize] = useState(0); + const [lastUpdated] = useState(''); useEffectOnce(() => { pilosa.get @@ -26,48 +24,6 @@ export const MoleculaTablesContainer = () => { .then((res) => setTables(res.data.indexes)) .catch((err) => console.log(err)) ); - - pilosa.get.usage().then((res) => { - const nodes = Object.keys(res.data); - let data = {}; - nodes.forEach((node) => { - const nodeIndexes = res.data[node].diskUsage.indexes; - const indexList = Object.keys(nodeIndexes); - indexList.forEach((i) => { - const nodeData = nodeIndexes[i]; - if (data[i]) { - data[i] = { - total: data[i].total + nodeData.total, - fieldKeysTotal: data[i].fieldKeysTotal + nodeData.fieldKeysTotal, - indexKeys: data[i].indexKeys + nodeData.indexKeys, - fragments: data[i].fragments + nodeData.fragments, - metadata: data[i].metadata + nodeData.metadata, - fields: [...data[i].fields, nodeData.fields] - }; - } else { - data[i] = { - total: nodeData.total, - fieldKeysTotal: nodeData.fieldKeysTotal, - indexKeys: nodeData.indexKeys, - fragments: nodeData.fragments, - metadata: nodeData.metadata, - fields: [nodeData.fields] - }; - } - }); - - if(!lastUpdated) { - setLastUpdated(res.data[node].lastUpdated); - } - }); - - const sorted = OrderBy(data, ['total'], ['desc']); - if (sorted.length > 0) { - setMaxSize(sorted[0].total); - } - - setDataDistribution(data); - }); }); useEffect(() => { @@ -85,21 +41,10 @@ export const MoleculaTablesContainer = () => { }, [match, tables, history]); return selectedTable ? ( - + ) : ( diff --git a/lattice/src/App/MoleculaTables/UsageBreakdown/UsageBreakdown.module.scss b/lattice/src/App/MoleculaTables/UsageBreakdown/UsageBreakdown.module.scss deleted file mode 100644 index 6e3cf558f..000000000 --- a/lattice/src/App/MoleculaTables/UsageBreakdown/UsageBreakdown.module.scss +++ /dev/null @@ -1,62 +0,0 @@ -.label { - font-size: 0.75rem; - color: var(--text-secondary); - margin-bottom: 4px; - font-weight: 400; -} - -.usageBreakdown { - display: flex; - align-items: center; - - .usageBreakdownLabel { - white-space: nowrap; - margin-right: 8px; - - &.smallLabel { - font-size: 12px; - } - } -} - -.breakdown { - display: flex; - align-items: center; - height: 13px; - border-radius: 4px; - background: rgba(var(--contrast-rgb), 0.1); - - .fieldKeysTotal { - height: 13px; - background: rgba(88, 80, 141, 0.7); - } - - .indexKeys { - height: 13px; - background: rgba(255, 99, 97, 0.7); - } - - .keys { - height: 13px; - background: rgba(88, 80, 141, 0.7); - } - - .fragments { - height: 13px; - background: rgba(255, 166, 0, 0.7); - } - - .metadata { - height: 13px; - background: rgba(188, 80, 144, 0.7); - } - - .bar:first-child { - border-top-left-radius: 4px; - border-bottom-left-radius: 4px; - } - .bar:last-child { - border-top-right-radius: 4px; - border-bottom-right-radius: 4px; - } -} diff --git a/lattice/src/App/MoleculaTables/UsageBreakdown/UsageBreakdown.tsx b/lattice/src/App/MoleculaTables/UsageBreakdown/UsageBreakdown.tsx deleted file mode 100644 index 78cc13c3d..000000000 --- a/lattice/src/App/MoleculaTables/UsageBreakdown/UsageBreakdown.tsx +++ /dev/null @@ -1,183 +0,0 @@ -import React, { FC, Fragment } from 'react'; -import classNames from 'classnames'; -import Tooltip from '@material-ui/core/Tooltip'; -import Typography from '@material-ui/core/Typography'; -import { formatBytes } from 'shared/utils/formatBytes'; -import css from './UsageBreakdown.module.scss'; - -type UsageBreakdownProps = { - data: any; - width?: string; - showLabel?: boolean; - usageValueSize?: 'small' | 'medium'; -}; - -export const UsageBreakdown: FC = ({ - data = {}, - width, - showLabel = true, - usageValueSize = 'medium' -}) => { - const { - total, - fieldKeysTotal, - indexKeys, - fragments, - metadata, - keys, - uncached - } = data; - const fieldKeysPercentage = - fieldKeysTotal && total ? (fieldKeysTotal / total) * 100 : 0; - const indexKeysPercentage = indexKeys ? (indexKeys / total) * 100 : 0; - const fragmentsPercentage = fragments ? (fragments / total) * 100 : 0; - const metadataPercentage = metadata ? (metadata / total) * 100 : 0; - const keysPercentage = keys && total ? (keys / total) * 100 : 0; - - return ( - - {showLabel ? : null} -
- {total ? ( - - - {formatBytes(total)} - -
- {fieldKeysTotal ? ( - - - - {formatBytes(fieldKeysTotal)} ( - {fieldKeysPercentage.toLocaleString(undefined, { - maximumFractionDigits: 1 - })} - %) - - - } - placement="top" - arrow - > -
- - ) : null} - {indexKeys ? ( - - - - {formatBytes(indexKeys)} ( - {indexKeysPercentage.toLocaleString(undefined, { - maximumFractionDigits: 1 - })} - %) - - - } - placement="top" - arrow - > -
- - ) : null} - {keys ? ( - - - - {formatBytes(keys)} ( - {keysPercentage.toLocaleString(undefined, { - maximumFractionDigits: 1 - })} - %) - - - } - placement="top" - arrow - > -
- - ) : null} - {fragments ? ( - - - - {formatBytes(fragments)} ( - {fragmentsPercentage.toLocaleString(undefined, { - maximumFractionDigits: 1 - })} - %) - - - } - placement="top" - arrow - > -
- - ) : null} - {metadata ? ( - - - - {formatBytes(metadata)} ( - {metadataPercentage.toLocaleString(undefined, { - maximumFractionDigits: 1 - })} - %) - - - } - placement="top" - arrow - > -
- - ) : null} -
- - ) : uncached ? ( - - Waiting... - - ) : ( - - Calculating... - - )} -
- - ); -}; diff --git a/lattice/src/App/MoleculaTables/UsageBreakdown/index.ts b/lattice/src/App/MoleculaTables/UsageBreakdown/index.ts deleted file mode 100644 index 36362bf49..000000000 --- a/lattice/src/App/MoleculaTables/UsageBreakdown/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './UsageBreakdown'; diff --git a/lattice/src/App/Query/QueryContainer.tsx b/lattice/src/App/Query/QueryContainer.tsx index a26190eb0..45401a7ca 100644 --- a/lattice/src/App/Query/QueryContainer.tsx +++ b/lattice/src/App/Query/QueryContainer.tsx @@ -119,19 +119,7 @@ export const QueryContainer: FC<{}> = () => { setLoading(false); } } else { - let queryArr = query.split(' '); - queryArr.forEach((word, idx) => { - if (word.includes('-')) { - let wordArr = word.split('.'); - wordArr.forEach((section, idx) => { - if (section.includes('-') && !word.includes('`')) { - wordArr[idx] = `\`${wordArr[idx]}\``; - } - }); - queryArr[idx] = wordArr.join('.'); - } - }); - querySQL(queryArr.join(' '), handleQueryMessages, handleQueryEnd); + querySQL(query, handleQueryMessages, handleQueryEnd); } } }; diff --git a/lattice/src/services/eventServices.tsx b/lattice/src/services/eventServices.tsx index b2adcfd33..a3a56e189 100644 --- a/lattice/src/services/eventServices.tsx +++ b/lattice/src/services/eventServices.tsx @@ -42,9 +42,6 @@ export const pilosa = { metrics() { return api.get('/metrics.json'); }, - usage() { - return api.get('/ui/usage'); - }, queryHistory() { return api.get('/query-history'); }, diff --git a/pb/private.pb.go b/pb/private.pb.go index 3a2807420..21d27da12 100644 --- a/pb/private.pb.go +++ b/pb/private.pb.go @@ -93,6 +93,7 @@ type FieldOptions struct { Min *Decimal `protobuf:"bytes,17,opt,name=Min,proto3" json:"Min,omitempty"` Max *Decimal `protobuf:"bytes,18,opt,name=Max,proto3" json:"Max,omitempty"` TimeUnit string `protobuf:"bytes,19,opt,name=TimeUnit,proto3" json:"TimeUnit,omitempty"` + Ttl string `protobuf:"bytes,20,opt,name=Ttl,proto3" json:"Ttl,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -236,6 +237,13 @@ func (m *FieldOptions) GetTimeUnit() string { return "" } +func (m *FieldOptions) GetTtl() string { + if m != nil { + return m.Ttl + } + return "" +} + type ImportResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -2759,110 +2767,111 @@ func init() { func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1639 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdf, 0x6e, 0x1b, 0x45, - 0x17, 0xff, 0x76, 0xd7, 0x8e, 0xed, 0xe3, 0x38, 0x71, 0xa6, 0xf9, 0xfa, 0x6d, 0xd2, 0x7e, 0x91, - 0x33, 0xa0, 0xd6, 0x44, 0x22, 0x88, 0xf4, 0xa2, 0x08, 0x6e, 0x9a, 0xd8, 0x69, 0x31, 0x25, 0x6d, - 0x3a, 0x49, 0x73, 0x09, 0x9a, 0xd8, 0xa3, 0x64, 0x95, 0xf5, 0xae, 0xd9, 0x5d, 0xa7, 0x76, 0x2f, - 0x90, 0x40, 0x20, 0xb8, 0xe1, 0x9e, 0x2b, 0x9e, 0x81, 0x1b, 0xde, 0x81, 0x1b, 0x24, 0x1e, 0x01, - 0x95, 0x17, 0x41, 0x73, 0x66, 0x66, 0x77, 0xed, 0x3a, 0x35, 0x44, 0xdc, 0xed, 0xf9, 0x9d, 0x99, - 0xf3, 0x7f, 0xce, 0x9c, 0x59, 0xa8, 0x0d, 0x22, 0xef, 0x92, 0x27, 0x62, 0x7b, 0x10, 0x85, 0x49, - 0x48, 0xec, 0xc1, 0xe9, 0xfa, 0xe2, 0x60, 0x78, 0xea, 0x7b, 0x5d, 0x85, 0xd0, 0x47, 0x50, 0xe9, - 0x04, 0x3d, 0x31, 0x3a, 0x10, 0x09, 0x27, 0x04, 0x0a, 0x8f, 0xc5, 0x38, 0x76, 0x9d, 0x86, 0xd5, - 0x2c, 0x33, 0xfc, 0x26, 0x77, 0x60, 0xe9, 0x38, 0xe2, 0xdd, 0x8b, 0xfd, 0x91, 0x17, 0x27, 0x22, - 0xe8, 0x0a, 0xb7, 0x80, 0xdc, 0x29, 0x94, 0xfe, 0xec, 0xc0, 0xe2, 0x43, 0x4f, 0xf8, 0xbd, 0xa7, - 0x83, 0xc4, 0x0b, 0x83, 0x58, 0x0a, 0x3b, 0x1e, 0x0f, 0x84, 0x5b, 0x6e, 0x58, 0xcd, 0x0a, 0xc3, - 0x6f, 0x72, 0x1b, 0x2a, 0x2d, 0xde, 0x3d, 0x17, 0xc8, 0x70, 0x90, 0x91, 0x01, 0x29, 0xf7, 0xc8, - 0x7b, 0xa9, 0xb4, 0xd4, 0x58, 0x06, 0x90, 0x06, 0x54, 0x8f, 0xbd, 0xbe, 0x78, 0x36, 0xe4, 0x41, - 0x32, 0xec, 0xbb, 0x45, 0xdc, 0x9d, 0x87, 0xc8, 0x4d, 0x58, 0x78, 0xea, 0xf7, 0x0e, 0xbc, 0xc0, - 0xad, 0x34, 0xac, 0xa6, 0xc3, 0x34, 0x65, 0x70, 0x3e, 0x72, 0x21, 0xc3, 0xf9, 0x28, 0x75, 0xb7, - 0x3a, 0xe9, 0xee, 0x93, 0xf0, 0x28, 0xe1, 0x41, 0x8f, 0x47, 0xbd, 0x13, 0x4f, 0xbc, 0x70, 0x17, - 0x95, 0xbb, 0x93, 0xa8, 0xdc, 0xbb, 0xc7, 0x63, 0xe1, 0xd6, 0x50, 0x22, 0x7e, 0x93, 0x75, 0x28, - 0xef, 0x79, 0x49, 0x5b, 0x0c, 0x92, 0x73, 0x77, 0xa9, 0x61, 0x35, 0x0b, 0x2c, 0xa5, 0xc9, 0x2a, - 0x14, 0x8f, 0xba, 0xdc, 0x17, 0xee, 0x32, 0x6e, 0x50, 0x04, 0xa1, 0xb0, 0xf8, 0x30, 0x8c, 0x84, - 0x77, 0x16, 0x60, 0x12, 0xdc, 0x3a, 0x3a, 0x35, 0x81, 0x91, 0xff, 0x83, 0x23, 0x5d, 0x5a, 0x69, - 0x58, 0xcd, 0xea, 0x4e, 0x75, 0x7b, 0x70, 0xba, 0xdd, 0x16, 0x5d, 0xaf, 0xcf, 0x7d, 0x26, 0x71, - 0x64, 0xf3, 0x91, 0x4b, 0x66, 0xb1, 0xf9, 0x48, 0xda, 0x24, 0x43, 0xf4, 0x3c, 0xf0, 0x12, 0xf7, - 0x06, 0x4a, 0x4f, 0x69, 0x4a, 0x61, 0xa9, 0xd3, 0x1f, 0x84, 0x51, 0xc2, 0x44, 0x3c, 0x08, 0x83, - 0x58, 0x90, 0x3a, 0x38, 0xfb, 0x51, 0xe4, 0x5a, 0xb8, 0x50, 0x7e, 0xd2, 0x2f, 0xa1, 0xbe, 0xe7, - 0x87, 0xdd, 0x8b, 0x36, 0x4f, 0x38, 0x13, 0x5f, 0x0c, 0x45, 0x9c, 0x48, 0x5f, 0x94, 0xb9, 0x6a, - 0x9d, 0x22, 0x24, 0x8a, 0xf9, 0x77, 0x6d, 0x85, 0x22, 0x21, 0xe3, 0x84, 0x51, 0x54, 0xe9, 0xc2, - 0x6f, 0x8c, 0xc5, 0x39, 0x8f, 0x7a, 0x98, 0xe3, 0x02, 0x53, 0x84, 0x44, 0x51, 0x13, 0xd6, 0x45, - 0x81, 0x29, 0x82, 0x76, 0x60, 0x25, 0xa7, 0x5f, 0x9b, 0x79, 0x13, 0x16, 0x58, 0xf8, 0xa2, 0xd3, - 0x8e, 0x5d, 0xab, 0xe1, 0x34, 0x0b, 0x4c, 0x53, 0x58, 0x40, 0xa1, 0x3f, 0xec, 0x07, 0x92, 0x65, - 0x23, 0x2b, 0x03, 0xe8, 0x1a, 0x14, 0xb1, 0x9a, 0xa4, 0x97, 0xd9, 0x5e, 0xf9, 0x49, 0xbf, 0xb2, - 0xa0, 0x72, 0xc0, 0x47, 0x68, 0x48, 0x4c, 0xee, 0x43, 0xd9, 0xe4, 0x1a, 0x17, 0x55, 0x77, 0x6e, - 0xc9, 0xb8, 0xa6, 0x0b, 0xb6, 0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x17, 0xaf, 0x7f, 0x04, - 0xb5, 0x09, 0x96, 0xd4, 0x74, 0x21, 0xc6, 0x26, 0x9e, 0x17, 0x62, 0x2c, 0xbd, 0xbc, 0xe4, 0xfe, - 0x50, 0x60, 0x94, 0x0a, 0x4c, 0x11, 0x1f, 0xda, 0x1f, 0x58, 0xf4, 0x04, 0x48, 0x2b, 0x12, 0x3c, - 0x11, 0xa8, 0xe4, 0x40, 0xc4, 0x31, 0x3f, 0x13, 0xf3, 0x62, 0xed, 0xe4, 0x63, 0x9d, 0xc6, 0xd5, - 0xce, 0xc5, 0x95, 0x6e, 0x01, 0x69, 0x0b, 0x5f, 0x24, 0x42, 0x9f, 0xf3, 0x37, 0xc8, 0xa5, 0x17, - 0xc6, 0x86, 0xf9, 0x6b, 0xc9, 0x26, 0x14, 0x64, 0xd3, 0x40, 0x65, 0xd5, 0x9d, 0x9a, 0x8c, 0x50, - 0xda, 0x49, 0x18, 0xb2, 0x30, 0x1f, 0x28, 0xae, 0xb7, 0x9b, 0xa0, 0xa9, 0x0e, 0xcb, 0x00, 0xfa, - 0x8d, 0x65, 0xb4, 0xa1, 0xf9, 0x7f, 0xd3, 0xe3, 0x89, 0xea, 0x7a, 0x5b, 0xdb, 0xe0, 0xa0, 0x0d, - 0x75, 0x69, 0x43, 0xbe, 0x07, 0xcd, 0x32, 0xa3, 0x30, 0x6d, 0xc6, 0x03, 0x13, 0x9f, 0xeb, 0x5a, - 0x41, 0xbb, 0x70, 0x4b, 0x49, 0xd8, 0xbd, 0xe4, 0x9e, 0xcf, 0x4f, 0xfd, 0x7f, 0x94, 0xc2, 0x09, - 0x87, 0x5c, 0x28, 0xe1, 0xde, 0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09, - 0xef, 0x0b, 0x2d, 0x0d, 0xbf, 0xd3, 0x38, 0xd8, 0x6f, 0x8c, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93, - 0xfd, 0xdd, 0x91, 0x2a, 0x91, 0x98, 0x13, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c, - 0xbc, 0x05, 0x25, 0xb4, 0x5c, 0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x94, 0x33, 0xc3, 0xa1, 0xdf, 0x5a, - 0xda, 0xd9, 0x99, 0x66, 0x4e, 0xa8, 0xb2, 0xa7, 0x54, 0x91, 0xbb, 0x50, 0xd2, 0xf6, 0x62, 0xb7, - 0x78, 0xad, 0xa6, 0x0c, 0x97, 0x6c, 0xc2, 0x02, 0x7a, 0x17, 0xbb, 0x85, 0xcc, 0x10, 0x44, 0x98, - 0x66, 0xd0, 0x7d, 0x70, 0x9e, 0xb3, 0x8e, 0x6c, 0x14, 0x68, 0xbd, 0x31, 0x43, 0x53, 0xd2, 0xb8, - 0x8f, 0xc3, 0x38, 0xd1, 0xb1, 0xc7, 0x6f, 0x89, 0x1d, 0x86, 0x91, 0xaa, 0xd3, 0x1a, 0xc3, 0x6f, - 0xfa, 0xbd, 0x05, 0x85, 0x27, 0x61, 0x4f, 0x90, 0x25, 0xb0, 0x3b, 0x6d, 0x2d, 0xc4, 0xee, 0xb4, - 0xc9, 0x1a, 0xca, 0xd7, 0xf1, 0x2e, 0x49, 0xfd, 0xcf, 0x59, 0x87, 0xa1, 0xce, 0xdb, 0x50, 0xe9, - 0xc4, 0x87, 0x91, 0xd7, 0xe7, 0xd1, 0x58, 0xdf, 0xa4, 0x19, 0x80, 0x67, 0x34, 0xe1, 0x89, 0xba, - 0xdf, 0x2a, 0x4c, 0x11, 0x64, 0x13, 0x4a, 0x8f, 0xd8, 0x61, 0x4b, 0x8a, 0x2c, 0x4e, 0x8a, 0x34, - 0x38, 0x7d, 0x00, 0x75, 0x69, 0x09, 0xae, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x5a, 0xa6, - 0xa9, 0x4c, 0x89, 0x9d, 0x53, 0x42, 0x1f, 0x2a, 0x09, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89, - 0x34, 0x0a, 0xa8, 0x31, 0x45, 0x90, 0xdb, 0xca, 0x6b, 0xed, 0x5e, 0x59, 0xda, 0x22, 0x69, 0x86, - 0x28, 0x1d, 0x03, 0x18, 0x4b, 0x86, 0x71, 0xba, 0xd6, 0x9a, 0xb5, 0x96, 0x50, 0x53, 0x3e, 0xfa, - 0x88, 0x82, 0xe4, 0x2b, 0x84, 0x99, 0xc2, 0x7a, 0x27, 0x2b, 0x2c, 0x95, 0xcf, 0xe5, 0x34, 0xef, - 0x4a, 0x47, 0x56, 0x5e, 0xe7, 0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0x26, - 0x0c, 0x11, 0x2d, 0x4c, 0xb3, 0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x4d, 0x33, 0x35, 0x35, 0x61, - 0x79, 0xf2, 0xc0, 0x9b, 0x3b, 0x67, 0x1a, 0x9e, 0xa3, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18, - 0x27, 0x22, 0x4a, 0x63, 0x5a, 0xd1, 0x40, 0x9a, 0xda, 0x0c, 0x98, 0x9d, 0x5d, 0xb2, 0x01, 0x45, - 0x19, 0x71, 0x75, 0xb8, 0xf3, 0x89, 0x50, 0x70, 0x2e, 0x13, 0x85, 0xab, 0x32, 0x41, 0x4f, 0xa0, - 0xbc, 0x77, 0xd4, 0x79, 0x14, 0x85, 0xc3, 0xc1, 0x4c, 0x8f, 0xcd, 0x48, 0x67, 0xe7, 0x46, 0xba, - 0xba, 0x1a, 0x4f, 0x94, 0x57, 0x38, 0x91, 0xd4, 0xd5, 0x44, 0x52, 0xd0, 0x08, 0x1f, 0xd1, 0x23, - 0x58, 0x51, 0xee, 0xca, 0x8e, 0x73, 0x9d, 0xb6, 0x68, 0xa6, 0x08, 0x27, 0x9b, 0x22, 0xa4, 0x50, - 0xd5, 0x75, 0xff, 0x4d, 0xa1, 0xbf, 0xd9, 0xb0, 0xc2, 0x44, 0xec, 0xbd, 0x14, 0x9d, 0x20, 0x4e, - 0xa2, 0x61, 0x57, 0x76, 0x1c, 0xb9, 0xff, 0x93, 0xf0, 0x54, 0xe7, 0xc2, 0x61, 0x8a, 0x78, 0xf3, - 0x29, 0x21, 0x14, 0x4a, 0xf9, 0x26, 0x90, 0x5f, 0x60, 0x18, 0x64, 0x0b, 0x4a, 0x47, 0xe1, 0x30, - 0xea, 0xa6, 0x95, 0x8f, 0x9d, 0x5b, 0xe9, 0x57, 0x0c, 0x66, 0x16, 0x90, 0xc7, 0x40, 0x8e, 0x23, - 0x1e, 0xc4, 0x3e, 0x97, 0x26, 0x99, 0x6d, 0xe5, 0x6c, 0x3c, 0xc9, 0x71, 0x27, 0x24, 0xcc, 0xd8, - 0x46, 0xb6, 0xf3, 0x47, 0xd8, 0x2d, 0xa1, 0x7d, 0x4b, 0xc6, 0x3e, 0x7d, 0x4e, 0xf2, 0x87, 0xfc, - 0xfe, 0x54, 0x85, 0xba, 0x0b, 0xb8, 0x65, 0x45, 0x6e, 0x99, 0x60, 0xb0, 0xc9, 0x75, 0xf4, 0x6b, - 0x0b, 0x16, 0xf3, 0xd6, 0xcc, 0x69, 0x17, 0x69, 0xfa, 0xec, 0xf9, 0xd3, 0x8e, 0x49, 0x5f, 0x61, - 0xd6, 0x64, 0x59, 0xcc, 0x4f, 0x40, 0x21, 0xfc, 0xef, 0x8a, 0xe0, 0x5c, 0xcb, 0x9c, 0x06, 0x54, - 0x0f, 0x79, 0x94, 0x78, 0x52, 0x98, 0xbe, 0xa7, 0x8b, 0x2c, 0x0f, 0x51, 0x01, 0x6b, 0xaf, 0x15, - 0x51, 0x2b, 0xec, 0x0f, 0x64, 0xb5, 0x5e, 0xab, 0x98, 0x64, 0x9b, 0x8e, 0xa2, 0x30, 0x32, 0x11, - 0x40, 0x82, 0xee, 0x41, 0xf9, 0x38, 0x1c, 0x84, 0x7e, 0x78, 0x36, 0x9e, 0xd3, 0x32, 0x5c, 0x28, - 0xa9, 0xab, 0x41, 0xb5, 0xa8, 0x0a, 0x33, 0x24, 0xbd, 0x21, 0xeb, 0xbd, 0xcb, 0xfd, 0xee, 0xd0, - 0xe7, 0x89, 0xc0, 0xf9, 0x18, 0xc1, 0x4f, 0x43, 0xde, 0x53, 0x5d, 0x41, 0x1f, 0x2d, 0xfa, 0xb9, - 0x2e, 0x40, 0x8e, 0xee, 0xe4, 0xae, 0xa0, 0x5d, 0x04, 0xcc, 0x15, 0xa4, 0x28, 0xf2, 0x3e, 0x54, - 0x73, 0xab, 0xb5, 0x5b, 0xcb, 0x69, 0x9d, 0x2a, 0x98, 0xe5, 0xd7, 0xd0, 0x5f, 0xac, 0x89, 0x3d, - 0xaf, 0xdd, 0xb9, 0x5a, 0xd5, 0xa5, 0x0a, 0x52, 0x99, 0x69, 0x4a, 0xba, 0xbe, 0x3f, 0xea, 0xfa, - 0xc3, 0x58, 0xb2, 0xf4, 0x85, 0x9b, 0x02, 0xd2, 0x75, 0xf9, 0xe0, 0x09, 0x87, 0x66, 0xb8, 0x31, - 0xa4, 0x7c, 0x1a, 0xb5, 0x05, 0xef, 0xf9, 0x5e, 0x20, 0xb0, 0x5e, 0x1c, 0x96, 0xd2, 0x64, 0x4b, - 0xf5, 0x58, 0x53, 0xe8, 0xab, 0x53, 0x86, 0x23, 0x4f, 0x75, 0xde, 0x98, 0x12, 0xa8, 0x4f, 0xb3, - 0xe8, 0x2a, 0x10, 0x55, 0x01, 0xbb, 0xa7, 0x61, 0x64, 0x6e, 0x5b, 0xda, 0x32, 0xcd, 0x45, 0x46, - 0x7f, 0xde, 0x25, 0x9e, 0x45, 0xd6, 0xce, 0x47, 0x96, 0x7e, 0x06, 0x4b, 0x7a, 0xb6, 0x13, 0x11, - 0x16, 0xb4, 0x0c, 0x00, 0x13, 0xdd, 0x50, 0x8e, 0x89, 0xe6, 0x55, 0x93, 0x01, 0x52, 0xce, 0x89, - 0x7c, 0x64, 0x98, 0xdb, 0x49, 0x53, 0x38, 0x1b, 0x79, 0x67, 0x81, 0xe8, 0xe1, 0x8d, 0xe1, 0x30, - 0x4d, 0xd1, 0x1f, 0x6c, 0x58, 0x55, 0x43, 0x67, 0x70, 0x26, 0xe2, 0x24, 0x53, 0x23, 0x9f, 0xd1, - 0x03, 0xec, 0xff, 0xda, 0x50, 0x45, 0xc9, 0x27, 0x73, 0xcb, 0x17, 0x3c, 0xca, 0x6c, 0x50, 0x8a, - 0xa6, 0x50, 0x79, 0x6e, 0x10, 0xd1, 0xd7, 0xb3, 0x1a, 0x42, 0xf3, 0x10, 0xd9, 0x83, 0xb2, 0x76, - 0xcd, 0x34, 0xc4, 0x3b, 0x78, 0x4b, 0xcd, 0xb0, 0xc6, 0xcc, 0xb7, 0xb1, 0x7e, 0x83, 0x19, 0x72, - 0xfd, 0x29, 0xd4, 0x26, 0x58, 0x33, 0xde, 0x60, 0xcd, 0xfc, 0x1b, 0xac, 0xba, 0x43, 0x72, 0xe3, - 0xb2, 0x96, 0x9e, 0x7f, 0x97, 0xb5, 0xe0, 0xbf, 0xb3, 0x0c, 0x88, 0xc9, 0x16, 0x38, 0xd2, 0x50, - 0x35, 0x0c, 0xbb, 0x57, 0x19, 0xca, 0xe4, 0x22, 0xfa, 0x93, 0xa5, 0x83, 0x2a, 0x34, 0xdf, 0xbc, - 0xa5, 0xef, 0xe5, 0x85, 0x6c, 0xa6, 0x42, 0xa6, 0x96, 0x6d, 0xa7, 0x8e, 0xca, 0xd5, 0xeb, 0xcf, - 0xa0, 0x3c, 0xcb, 0xbd, 0x82, 0x72, 0xef, 0xbd, 0x49, 0xf7, 0xd6, 0xae, 0xb2, 0x2c, 0xce, 0x79, - 0xb9, 0x57, 0xff, 0xf5, 0xd5, 0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f, - 0x7f, 0x6e, 0xfc, 0xe7, 0x74, 0x01, 0x7f, 0x10, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x38, - 0x55, 0x86, 0x89, 0x43, 0x12, 0x00, 0x00, + // 1650 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xcf, 0x72, 0x1b, 0x45, + 0x13, 0xff, 0x76, 0x57, 0xb2, 0xa4, 0x96, 0x65, 0xcb, 0x13, 0x7f, 0xf9, 0xd6, 0x4e, 0x3e, 0x97, + 0x3c, 0x50, 0x89, 0x70, 0x15, 0xa6, 0x70, 0x0e, 0xa1, 0xe0, 0x12, 0x5b, 0x72, 0x82, 0x08, 0x4e, + 0x9c, 0xb1, 0xe3, 0x23, 0xd4, 0x58, 0x9a, 0xb2, 0xb7, 0xbc, 0xda, 0x15, 0xbb, 0x2b, 0x47, 0xca, + 0x81, 0x2a, 0x28, 0x28, 0xb8, 0x70, 0xe7, 0xc4, 0x5b, 0xf0, 0x02, 0x9c, 0xb8, 0x50, 0xc5, 0x23, + 0x50, 0xe1, 0x45, 0xa8, 0xe9, 0x99, 0xd9, 0x5d, 0x29, 0x72, 0x04, 0x2e, 0x6e, 0xdb, 0xbf, 0xee, + 0xe9, 0x7f, 0xd3, 0xd3, 0xd3, 0xb3, 0x50, 0x1b, 0x44, 0xde, 0x25, 0x4f, 0xc4, 0xf6, 0x20, 0x0a, + 0x93, 0x90, 0xd8, 0x83, 0xd3, 0xf5, 0xc5, 0xc1, 0xf0, 0xd4, 0xf7, 0xba, 0x0a, 0xa1, 0x8f, 0xa0, + 0xd2, 0x09, 0x7a, 0x62, 0x74, 0x20, 0x12, 0x4e, 0x08, 0x14, 0x1e, 0x8b, 0x71, 0xec, 0x3a, 0x0d, + 0xab, 0x59, 0x66, 0xf8, 0x4d, 0xee, 0xc0, 0xd2, 0x71, 0xc4, 0xbb, 0x17, 0xfb, 0x23, 0x2f, 0x4e, + 0x44, 0xd0, 0x15, 0x6e, 0x01, 0xb9, 0x53, 0x28, 0xfd, 0xc5, 0x81, 0xc5, 0x87, 0x9e, 0xf0, 0x7b, + 0x4f, 0x07, 0x89, 0x17, 0x06, 0xb1, 0x54, 0x76, 0x3c, 0x1e, 0x08, 0xb7, 0xdc, 0xb0, 0x9a, 0x15, + 0x86, 0xdf, 0xe4, 0x36, 0x54, 0x5a, 0xbc, 0x7b, 0x2e, 0x90, 0xe1, 0x20, 0x23, 0x03, 0x52, 0xee, + 0x91, 0xf7, 0x52, 0x59, 0xa9, 0xb1, 0x0c, 0x20, 0x0d, 0xa8, 0x1e, 0x7b, 0x7d, 0xf1, 0x6c, 0xc8, + 0x83, 0x64, 0xd8, 0x77, 0x8b, 0xb8, 0x3a, 0x0f, 0x91, 0x9b, 0xb0, 0xf0, 0xd4, 0xef, 0x1d, 0x78, + 0x81, 0x5b, 0x69, 0x58, 0x4d, 0x87, 0x69, 0xca, 0xe0, 0x7c, 0xe4, 0x42, 0x86, 0xf3, 0x51, 0x1a, + 0x6e, 0x75, 0x32, 0xdc, 0x27, 0xe1, 0x51, 0xc2, 0x83, 0x1e, 0x8f, 0x7a, 0x27, 0x9e, 0x78, 0xe1, + 0x2e, 0xaa, 0x70, 0x27, 0x51, 0xb9, 0x76, 0x8f, 0xc7, 0xc2, 0xad, 0xa1, 0x46, 0xfc, 0x26, 0xeb, + 0x50, 0xde, 0xf3, 0x92, 0xb6, 0x18, 0x24, 0xe7, 0xee, 0x52, 0xc3, 0x6a, 0x16, 0x58, 0x4a, 0x93, + 0x55, 0x28, 0x1e, 0x75, 0xb9, 0x2f, 0xdc, 0x65, 0x5c, 0xa0, 0x08, 0x42, 0x61, 0xf1, 0x61, 0x18, + 0x09, 0xef, 0x2c, 0xc0, 0x4d, 0x70, 0xeb, 0x18, 0xd4, 0x04, 0x46, 0xfe, 0x0f, 0x8e, 0x0c, 0x69, + 0xa5, 0x61, 0x35, 0xab, 0x3b, 0xd5, 0xed, 0xc1, 0xe9, 0x76, 0x5b, 0x74, 0xbd, 0x3e, 0xf7, 0x99, + 0xc4, 0x91, 0xcd, 0x47, 0x2e, 0x99, 0xc5, 0xe6, 0x23, 0xe9, 0x93, 0x4c, 0xd1, 0xf3, 0xc0, 0x4b, + 0xdc, 0x1b, 0xa8, 0x3d, 0xa5, 0x49, 0x1d, 0x9c, 0xe3, 0xc4, 0x77, 0x57, 0x11, 0x96, 0x9f, 0x94, + 0xc2, 0x52, 0xa7, 0x3f, 0x08, 0xa3, 0x84, 0x89, 0x78, 0x10, 0x06, 0xb1, 0x90, 0x32, 0xfb, 0x51, + 0xe4, 0x5a, 0x4a, 0x66, 0x3f, 0x8a, 0xe8, 0x97, 0x50, 0xdf, 0xf3, 0xc3, 0xee, 0x45, 0x9b, 0x27, + 0x9c, 0x89, 0x2f, 0x86, 0x22, 0x4e, 0x64, 0x74, 0x2a, 0x00, 0x25, 0xa7, 0x08, 0x89, 0x62, 0x45, + 0xb8, 0xb6, 0x42, 0x91, 0x90, 0x99, 0xc3, 0xbc, 0xaa, 0x0d, 0xc4, 0x6f, 0xcc, 0xce, 0x39, 0x8f, + 0x7a, 0xb8, 0xeb, 0x05, 0xa6, 0x08, 0x89, 0xa2, 0x25, 0xac, 0x94, 0x02, 0x53, 0x04, 0xed, 0xc0, + 0x4a, 0xce, 0xbe, 0x76, 0xf3, 0x26, 0x2c, 0xb0, 0xf0, 0x45, 0xa7, 0x1d, 0xbb, 0x56, 0xc3, 0x69, + 0x16, 0x98, 0xa6, 0xb0, 0xa4, 0x42, 0x7f, 0xd8, 0x0f, 0x24, 0xcb, 0x46, 0x56, 0x06, 0xd0, 0x35, + 0x28, 0x62, 0x7d, 0xc9, 0x28, 0xb3, 0xb5, 0xf2, 0x93, 0x7e, 0x65, 0x41, 0xe5, 0x80, 0x8f, 0xd0, + 0x91, 0x98, 0xdc, 0x87, 0xb2, 0xd9, 0x7d, 0x14, 0xaa, 0xee, 0xdc, 0x92, 0x99, 0x4e, 0x05, 0xb6, + 0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x85, 0xd7, 0x3f, 0x82, 0xda, 0x04, 0x4b, 0x5a, 0xba, + 0x10, 0x63, 0x93, 0xcf, 0x0b, 0x31, 0x96, 0x51, 0x5e, 0x72, 0x7f, 0x28, 0x30, 0x4b, 0x05, 0xa6, + 0x88, 0x0f, 0xed, 0x0f, 0x2c, 0x7a, 0x02, 0xa4, 0x15, 0x09, 0x9e, 0x08, 0x34, 0x72, 0x20, 0xe2, + 0x98, 0x9f, 0x89, 0x79, 0xb9, 0x76, 0xf2, 0xb9, 0x4e, 0xf3, 0x6a, 0xe7, 0xf2, 0x4a, 0xb7, 0x80, + 0xb4, 0x85, 0x2f, 0x12, 0xa1, 0x4f, 0xfe, 0x1b, 0xf4, 0xd2, 0x0b, 0xe3, 0xc3, 0x7c, 0x59, 0xb2, + 0x09, 0x05, 0xd9, 0x46, 0xd0, 0x58, 0x75, 0xa7, 0x26, 0x33, 0x94, 0xf6, 0x16, 0x86, 0x2c, 0xdc, + 0x0f, 0x54, 0xd7, 0xdb, 0x4d, 0xd0, 0x55, 0x87, 0x65, 0x00, 0xfd, 0xc6, 0x32, 0xd6, 0xd0, 0xfd, + 0xbf, 0x19, 0xf1, 0x44, 0x75, 0xbd, 0xad, 0x7d, 0x70, 0xd0, 0x87, 0xba, 0xf4, 0x21, 0xdf, 0x95, + 0x66, 0xb9, 0x51, 0x98, 0x76, 0xe3, 0x81, 0xc9, 0xcf, 0x75, 0xbd, 0xa0, 0x5d, 0xb8, 0xa5, 0x34, + 0xec, 0x5e, 0x72, 0xcf, 0xe7, 0xa7, 0xfe, 0x3f, 0xda, 0xc2, 0x89, 0x80, 0x5c, 0x28, 0xe1, 0xda, + 0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09, 0xef, 0x0b, 0xad, 0x0d, 0xbf, + 0xd3, 0x3c, 0xd8, 0x6f, 0xcc, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93, 0x1d, 0xdf, 0x91, 0x26, 0x91, + 0x98, 0x93, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c, 0xbc, 0x05, 0x25, 0xf4, 0x5c, + 0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x2d, 0x67, 0x86, 0x43, 0xbf, 0xb5, 0x74, 0xb0, 0x33, 0xdd, 0x9c, + 0x30, 0x65, 0x4f, 0x99, 0x22, 0x77, 0xa1, 0xa4, 0xfd, 0xc5, 0x6e, 0xf1, 0x5a, 0x4d, 0x19, 0x2e, + 0xd9, 0x84, 0x05, 0x8c, 0x2e, 0x76, 0x0b, 0x99, 0x23, 0x88, 0x30, 0xcd, 0xa0, 0xfb, 0xe0, 0x3c, + 0x67, 0x1d, 0xd9, 0x28, 0xd0, 0x7b, 0xe3, 0x86, 0xa6, 0xa4, 0x73, 0x1f, 0x87, 0x71, 0xa2, 0x73, + 0x8f, 0xdf, 0x12, 0x3b, 0x0c, 0x23, 0x55, 0xa7, 0x35, 0x86, 0xdf, 0xf4, 0x7b, 0x0b, 0x0a, 0x4f, + 0xc2, 0x9e, 0x20, 0x4b, 0x60, 0x77, 0xda, 0x5a, 0x89, 0xdd, 0x69, 0x93, 0x35, 0xd4, 0xaf, 0xf3, + 0x5d, 0x92, 0xf6, 0x9f, 0xb3, 0x0e, 0x43, 0x9b, 0xb7, 0xa1, 0xd2, 0x89, 0x0f, 0x23, 0xaf, 0xcf, + 0xa3, 0xb1, 0xbe, 0x5b, 0x33, 0x00, 0xcf, 0x68, 0xc2, 0x13, 0x75, 0xe3, 0x55, 0x98, 0x22, 0xc8, + 0x26, 0x94, 0x1e, 0xb1, 0xc3, 0x96, 0x54, 0x59, 0x9c, 0x54, 0x69, 0x70, 0xfa, 0x00, 0xea, 0xd2, + 0x13, 0x94, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x7a, 0xa6, 0xa9, 0xcc, 0x88, 0x9d, 0x33, + 0x42, 0x1f, 0x2a, 0x0d, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89, 0x34, 0x2a, 0xa8, 0x31, 0x45, + 0x90, 0xdb, 0x2a, 0x6a, 0x1d, 0x5e, 0x59, 0xfa, 0x22, 0x69, 0x86, 0x28, 0x1d, 0x03, 0x18, 0x4f, + 0x86, 0x71, 0x2a, 0x6b, 0xcd, 0x92, 0x25, 0xd4, 0x94, 0x8f, 0x3e, 0xa2, 0x20, 0xf9, 0x0a, 0x61, + 0xa6, 0xb0, 0xde, 0xc9, 0x0a, 0x4b, 0xed, 0xe7, 0x72, 0xba, 0xef, 0xca, 0x46, 0x56, 0x5e, 0xe7, + 0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0xa6, 0x0c, 0x11, 0xad, 0x4c, 0xb3, + 0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x45, 0x33, 0x2d, 0x35, 0x61, 0x79, 0xf2, 0xc0, 0x9b, 0x3b, + 0x67, 0x1a, 0x9e, 0x63, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18, 0x27, 0x22, 0x4a, 0x73, 0x5a, + 0xd1, 0x40, 0xba, 0xb5, 0x19, 0x30, 0x7b, 0x77, 0xc9, 0x06, 0x14, 0x65, 0xc6, 0xd5, 0xe1, 0xce, + 0x6f, 0x84, 0x82, 0x73, 0x3b, 0x51, 0xb8, 0x6a, 0x27, 0xe8, 0x09, 0x94, 0xf7, 0x8e, 0x3a, 0x8f, + 0xa2, 0x70, 0x38, 0x98, 0x19, 0xb1, 0x19, 0xf2, 0xec, 0xdc, 0x90, 0x57, 0x57, 0x03, 0x8b, 0x8a, + 0x0a, 0x67, 0x94, 0xba, 0x9a, 0x51, 0x0a, 0x1a, 0xe1, 0x23, 0x7a, 0x04, 0x2b, 0x2a, 0x5c, 0xd9, + 0x71, 0xae, 0xd3, 0x16, 0xcd, 0x14, 0xe1, 0x64, 0x53, 0x84, 0x54, 0xaa, 0xba, 0xee, 0xbf, 0xa9, + 0xf4, 0x37, 0x1b, 0x56, 0x98, 0x88, 0xbd, 0x97, 0xa2, 0x13, 0xc4, 0x49, 0x34, 0xec, 0xca, 0x8e, + 0x23, 0xd7, 0x7f, 0x12, 0x9e, 0xea, 0xbd, 0x70, 0x98, 0x22, 0xde, 0x7c, 0x4a, 0x08, 0x85, 0x52, + 0xbe, 0x09, 0xe4, 0x05, 0x0c, 0x83, 0x6c, 0x41, 0xe9, 0x28, 0x1c, 0x46, 0xdd, 0xb4, 0xf2, 0xb1, + 0x73, 0x2b, 0xfb, 0x8a, 0xc1, 0x8c, 0x00, 0x79, 0x0c, 0xe4, 0x38, 0xe2, 0x41, 0xec, 0x73, 0xe9, + 0x92, 0x59, 0x56, 0xce, 0xc6, 0x93, 0x1c, 0x77, 0x42, 0xc3, 0x8c, 0x65, 0x64, 0x3b, 0x7f, 0x84, + 0xdd, 0x12, 0xfa, 0xb7, 0x64, 0xfc, 0xd3, 0xe7, 0x24, 0x7f, 0xc8, 0xef, 0x4f, 0x55, 0xa8, 0xbb, + 0x80, 0x4b, 0x56, 0xe4, 0x92, 0x09, 0x06, 0x9b, 0x94, 0xa3, 0x5f, 0x5b, 0xb0, 0x98, 0xf7, 0x66, + 0x4e, 0xbb, 0x48, 0xb7, 0xcf, 0x9e, 0x3f, 0xed, 0x98, 0xed, 0x2b, 0xcc, 0x9a, 0x2c, 0x8b, 0xf9, + 0x09, 0x28, 0x84, 0xff, 0x5d, 0x91, 0x9c, 0x6b, 0xb9, 0xd3, 0x80, 0xea, 0x21, 0x8f, 0x12, 0x4f, + 0x2a, 0xd3, 0xf7, 0x74, 0x91, 0xe5, 0x21, 0x2a, 0x60, 0xed, 0xb5, 0x22, 0x6a, 0x85, 0xfd, 0x81, + 0xac, 0xd6, 0x6b, 0x15, 0x93, 0x6c, 0xd3, 0x51, 0x14, 0x46, 0x26, 0x03, 0x48, 0xd0, 0x3d, 0x28, + 0x1f, 0x87, 0x83, 0xd0, 0x0f, 0xcf, 0xc6, 0x73, 0x5a, 0x86, 0x0b, 0x25, 0x75, 0x35, 0xa8, 0x16, + 0x55, 0x61, 0x86, 0xa4, 0x37, 0x64, 0xbd, 0x77, 0xb9, 0xdf, 0x1d, 0xfa, 0x3c, 0x11, 0x38, 0x1f, + 0x23, 0xf8, 0x69, 0xc8, 0x7b, 0xaa, 0x2b, 0xe8, 0xa3, 0x45, 0x3f, 0xd7, 0x05, 0xc8, 0x31, 0x9c, + 0xdc, 0x15, 0xb4, 0x8b, 0x80, 0xb9, 0x82, 0x14, 0x45, 0xde, 0x87, 0x6a, 0x4e, 0x5a, 0x87, 0xb5, + 0x9c, 0xd6, 0xa9, 0x82, 0x59, 0x5e, 0x86, 0xfe, 0x6c, 0x4d, 0xac, 0x79, 0xed, 0xce, 0xd5, 0xa6, + 0x2e, 0x55, 0x92, 0xca, 0x4c, 0x53, 0x32, 0xf4, 0xfd, 0x51, 0xd7, 0x1f, 0xc6, 0x92, 0xa5, 0x2f, + 0xdc, 0x14, 0x90, 0xa1, 0xcb, 0x27, 0x50, 0x38, 0x34, 0xc3, 0x8d, 0x21, 0xe5, 0x63, 0xa9, 0x2d, + 0x78, 0xcf, 0xf7, 0x02, 0x81, 0xf5, 0xe2, 0xb0, 0x94, 0x26, 0x5b, 0xaa, 0xc7, 0x9a, 0x42, 0x5f, + 0x9d, 0x72, 0x1c, 0x79, 0xaa, 0xf3, 0xc6, 0x94, 0x40, 0x7d, 0x9a, 0x45, 0x57, 0x81, 0xa8, 0x0a, + 0xd8, 0x3d, 0x0d, 0x23, 0x73, 0xdb, 0xd2, 0x96, 0x69, 0x2e, 0x32, 0xfb, 0xf3, 0x2e, 0xf1, 0x2c, + 0xb3, 0x76, 0x3e, 0xb3, 0xf4, 0x33, 0x58, 0xd2, 0xb3, 0x9d, 0x88, 0xb0, 0xa0, 0x65, 0x02, 0x98, + 0xe8, 0x86, 0x72, 0x4c, 0x34, 0xaf, 0x9a, 0x0c, 0x90, 0x7a, 0x4e, 0xe4, 0x23, 0xc3, 0xdc, 0x4e, + 0x9a, 0xc2, 0xd9, 0xc8, 0x3b, 0x0b, 0x44, 0x0f, 0x6f, 0x0c, 0x87, 0x69, 0x8a, 0xfe, 0x60, 0xc3, + 0xaa, 0x1a, 0x3a, 0x83, 0x33, 0x11, 0x27, 0x99, 0x19, 0xf9, 0xb0, 0x1e, 0x60, 0xff, 0xd7, 0x8e, + 0x2a, 0x4a, 0x3e, 0xa2, 0x5b, 0xbe, 0xe0, 0x51, 0xe6, 0x83, 0x32, 0x34, 0x85, 0xca, 0x73, 0x83, + 0x88, 0xbe, 0x9e, 0xd5, 0x10, 0x9a, 0x87, 0xc8, 0x1e, 0x94, 0x75, 0x68, 0xa6, 0x21, 0xde, 0xc1, + 0x5b, 0x6a, 0x86, 0x37, 0x66, 0xbe, 0x8d, 0xf5, 0x1b, 0xcc, 0x90, 0xeb, 0x4f, 0xa1, 0x36, 0xc1, + 0x9a, 0xf1, 0x06, 0x6b, 0xe6, 0xdf, 0x60, 0xd5, 0x1d, 0x92, 0x1b, 0x97, 0xb5, 0xf6, 0xfc, 0xbb, + 0xac, 0x05, 0xff, 0x9d, 0xe5, 0x40, 0x4c, 0xb6, 0xc0, 0x91, 0x8e, 0xaa, 0x61, 0xd8, 0xbd, 0xca, + 0x51, 0x26, 0x85, 0xe8, 0x4f, 0x96, 0x4e, 0xaa, 0xd0, 0x7c, 0xf3, 0x96, 0xbe, 0x97, 0x57, 0xb2, + 0x99, 0x2a, 0x99, 0x12, 0xdb, 0x4e, 0x03, 0x95, 0xd2, 0xeb, 0xcf, 0xa0, 0x3c, 0x2b, 0xbc, 0x82, + 0x0a, 0xef, 0xbd, 0xc9, 0xf0, 0xd6, 0xae, 0xf2, 0x2c, 0xce, 0x45, 0xb9, 0x57, 0xff, 0xf5, 0xd5, + 0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f, 0x7f, 0x6e, 0xfc, 0xe7, 0x74, + 0x01, 0x7f, 0x19, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0xcf, 0xdb, 0x63, 0xcd, 0x55, 0x12, + 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -2936,6 +2945,15 @@ func (m *FieldOptions) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.Ttl) > 0 { + i -= len(m.Ttl) + copy(dAtA[i:], m.Ttl) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.Ttl))) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa2 + } if len(m.TimeUnit) > 0 { i -= len(m.TimeUnit) copy(dAtA[i:], m.TimeUnit) @@ -5274,6 +5292,10 @@ func (m *FieldOptions) Size() (n int) { if l > 0 { n += 2 + l + sovPrivate(uint64(l)) } + l = len(m.Ttl) + if l > 0 { + n += 2 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6765,6 +6787,38 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { } m.TimeUnit = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 20: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ttl", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ttl = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) diff --git a/pb/private.proto b/pb/private.proto index 9f15939c7..9c4c37f25 100644 --- a/pb/private.proto +++ b/pb/private.proto @@ -25,6 +25,7 @@ message FieldOptions { Decimal Min = 17; Decimal Max = 18; string TimeUnit = 19; + string Ttl = 20; } message ImportResponse { diff --git a/pilosa.go b/pilosa.go index 9cf4f715f..218e37491 100644 --- a/pilosa.go +++ b/pilosa.go @@ -111,6 +111,12 @@ func newConflictError(err error) ConflictError { return ConflictError{err} } +// Unwrap makes it so that a ConflictError wrapping ErrFieldExists gets a +// true from errors.Is(ErrFieldExists). +func (c ConflictError) Unwrap() error { + return c.error +} + // NotFoundError wraps an error value to signify that a resource was not found // such that in an HTTP scenario, http.StatusNotFound would be returned. type NotFoundError error diff --git a/pql/parser.go b/pql/parser.go index 514870503..57f0d22ab 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -15,7 +15,7 @@ import ( // error strings in the parser const duplicateArgErrorMessage = "duplicate argument provided" const intOutOfRangeError = "integer is not in signed 64-bit range" -const invalidTimestampError = "string is not a timestamp" +const invalidTimestampError = "string is not a valid timestamp" // parser represents a parser for the PQL language. type parser struct { @@ -66,7 +66,7 @@ func (p *parser) Parse() (*Query, error) { if !ok { return nil, fmt.Errorf("unexpected parser error of type %T: %[1]v", v) } - if strings.HasPrefix(errorMessage, duplicateArgErrorMessage) || strings.HasPrefix(errorMessage, intOutOfRangeError) { + if strings.HasPrefix(errorMessage, duplicateArgErrorMessage) || strings.HasPrefix(errorMessage, intOutOfRangeError) || strings.HasPrefix(errorMessage, invalidTimestampError) { return nil, fmt.Errorf("%s", v) } else { panic(v) diff --git a/pql/parser_test.go b/pql/parser_test.go index 3cfa612a8..b829d6079 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -5,6 +5,7 @@ import ( "reflect" "strings" "testing" + "time" "github.com/molecula/featurebase/v3/pql" _ "github.com/molecula/featurebase/v3/test" @@ -197,6 +198,33 @@ func TestParser_Parse(t *testing.T) { } }) + t.Run("Timestamp", func(t *testing.T) { + twos := "2022-02-22T22:22:22Z" + date, err := time.Parse(time.RFC3339, twos) + if err != nil { + t.Fatal(err) + } + q, err := pql.ParseString(`Row(x>'2022-02-22T22:22:22Z')`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Row", + Args: map[string]interface{}{ + "x": &pql.Condition{Op: pql.GT, Value: date}, + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + q, err = pql.ParseString(`Row(x>'2024-04-24T24:24:24Z')`) + if err == nil { + t.Fatal("no error parsing invalid date") + } else if !strings.Contains(err.Error(), "not a valid timestamp") { + t.Fatalf("expected error for invalid timestamp, got: %s", err.Error()) + } + }) + t.Run("VariousSpaces", func(t *testing.T) { q, err := pql.ParseString(`TopN( x )`) if err != nil { diff --git a/qa/scripts/perf/able/able.yaml b/qa/scripts/perf/able/able.yaml new file mode 100644 index 000000000..07365f90b --- /dev/null +++ b/qa/scripts/perf/able/able.yaml @@ -0,0 +1,96 @@ +fields: + - name: "id" + type: uint # (default IDField (non-mutex)) + distribution: "sequential" + min: 0 + max: 1000000000 # 1B + repeat: false # if false, data generation stops when we hit >= max. only available with sequential + step: 1 + - name: "age" + type: int + distribution: "uniform" # uniform or zipfian # TODO should totally add some kind of poission, normal, gaussian, bimodal + min: 15 + max: 107 + null_chance: 0.01 + - name: "education_level" + type: string + source_file: "values/education.txt" + distribution: "zipfian" + s: 1.1 + v: 5.1 + - name: "gender" + type: string + source_file: "values/gender.txt" + distribution: "fixed" + - name: "income_bracket" + type: string + source_file: "values/income.txt" + - name: "domain" + type: "string-set" + min_num: 1 + max_num: 6 + source_file: "values/opendns-top-domains-10K.txt" + distribution: "zipfian" + s: 1.5 + v: 4.3 + - name: "timestamp" + type: "timestamp" # (default TimestampField) + min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano + max_date: 2010-01-02T15:04:05.001Z # RFC3339Nano + distribution: "increasing" # only "increasing" is supported right now + min_step_duration: "10us" + max_step_duration: "100ms" # generated values will add randomly between 1s and 1h to previous value starting at min_date. + repeat: false # stop at > max_date unless repeat=true... then go back to min. + - name: "political_party" + type: "string" + source_file: "values/political_parties.txt" + distribution: "zipfian" + s: 1.0001 + v: 1.0001 + - name: "ltv" + type: "float" # use idk_params to choose a scale + min_float: 0.2 + max_float: 1500 + distribution: "uniform" # only supported value + - name: "hobby" + type: "string-set" + source_file: "values/hobbies.txt" + distribution: "zipfian" + min_num: 0 + max_num: 4 + s: 1.3 + v: 2.5 + +# idk_params describe how data from "fields" should be ingested by IDK +idk_params: + primary_key_config: + field: "id" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up. + # fields is keyed by names of fields from top level "fields". It is + # not required that all fields appear here, those that don't will + # use the default ingestion. + fields: + id: + - type: "ID" + timestamp: + - type: "RecordTime" + layout: "2006-01-02T15:04:05Z" + epoch: 1970-01-01T00:00:00.0Z + name: "na" + domain: + - type: "StringArray" + time_quantum: "YMD" + ltv: + - type: "Decimal" + scale: 2 + income_bracket: + - type: "String" + mutex: true + education_level: + - type: "String" + mutex: true + gender: + - type: "String" + mutex: true + political_party: + - type: "String" + mutex: true diff --git a/qa/scripts/perf/able/ableRun.sh b/qa/scripts/perf/able/ableRun.sh new file mode 100644 index 000000000..92ca26987 --- /dev/null +++ b/qa/scripts/perf/able/ableRun.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# requires TF_VAR_cluster_prefix env var to be set +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "setting TF_VAR_cluster_prefix"; + export TF_VAR_cluster_prefix="able-$(openssl rand -base64 12 | tr -d /=+ | cut -c -16)" + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +$SCRIPT_DIR/ableSetup.sh +$SCRIPT_DIR/ableTest.sh +$SCRIPT_DIR/ableTeardown.sh diff --git a/qa/scripts/perf/able/ableSetup.sh b/qa/scripts/perf/able/ableSetup.sh new file mode 100755 index 000000000..8ba9a4f54 --- /dev/null +++ b/qa/scripts/perf/able/ableSetup.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# To run script: ./ableSetup.sh +export TF_IN_AUTOMATION=1 + +if [ -z ${TF_VAR_cluster_prefix+x} ]; then + echo "TF_VAR_cluster_prefix is unset"; + exit 1 +else + echo "TF_VAR_cluster_prefix is set to '$TF_VAR_cluster_prefix'"; +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source $SCRIPT_DIR/../../utilCluster.sh + +pushd ./qa/tf/perf/able +echo "Running terraform init..." +terraform init -input=false +echo "Running terraform apply..." +terraform apply -input=false -auto-approve +terraform output -json > outputs.json +popd + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + + +DEPLOYED_CLUSTER_PREFIX=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.cluster_prefix][0]["value"]') +echo "Using DEPLOYED_CLUSTER_PREFIX: ${DEPLOYED_CLUSTER_PREFIX}" + +DEPLOYED_CLUSTER_REPLICA_COUNT=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.fb_cluster_replica_count][0]["value"]') +echo "Using DEPLOYED_CLUSTER_REPLICA_COUNT: ${DEPLOYED_CLUSTDEPLOYED_CLUSTER_REPLICA_COUNTER_PREFIX}" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +DEPLOYED_INGEST_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][]') +echo "DEPLOYED_INGEST_IPS: {" +echo "${DEPLOYED_INGEST_IPS}" +echo "}" + +DEPLOYED_INGEST_IPS_LEN=`echo "$DEPLOYED_INGEST_IPS" | wc -l` + +#wait until we can connect to one of the hosts +for i in {0..24} +do + ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" + if [ $? -eq 0 ] + then + echo "Cluster is up after ${i} tries." + break + fi + sleep 10 +done + +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@${DATANODE0} "pwd" +if [ $? -ne 0 ] +then + echo "Unable to connect to cluster - giving up" + exit 1 +fi + +setupClusterNodes + +# verify featurebase running +echo "Verifying featurebase cluster running..." +curl -s http://${DATANODE0}:10101/status +if (( $? != 0 )) +then + echo "Featurebase cluster not running" + exit 1 +fi + +echo "Cluster running." + + + diff --git a/qa/scripts/perf/able/ableTeardown.sh b/qa/scripts/perf/able/ableTeardown.sh new file mode 100755 index 000000000..c95aae6b3 --- /dev/null +++ b/qa/scripts/perf/able/ableTeardown.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# To run script: ./ableTeardown.sh + +cd qa/tf/perf/able +export TF_IN_AUTOMATION=1 +terraform destroy -auto-approve diff --git a/qa/scripts/perf/able/ableTest.sh b/qa/scripts/perf/able/ableTest.sh new file mode 100755 index 000000000..d81ec2f51 --- /dev/null +++ b/qa/scripts/perf/able/ableTest.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# get the first ingest host +INGESTNODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.ingest_ips][0]["value"][0]') +echo "using INGESTNODE0 ${INGESTNODE0}" + +# get the first data host +DATANODE0=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][0]') +echo "using DATANODE0 ${DATANODE0}" + +# leaving this here because K6 is timing out and need to work out why +# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "wget https://github.com/grafana/k6/releases/download/v0.36.0/k6-v0.36.0-linux-arm64.tar.gz" +# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "tar -xvf k6-v0.36.0-linux-arm64.tar.gz" +# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "mkdir bin" +# ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "mv ./k6-v0.36.0-linux-arm64/k6 ./bin" + +echo "Copying tests to remote" +scp -r -i ~/.ssh/gitlab-featurebase-ci.pem ./qa/scripts/perf/able/*.js ec2-user@${INGESTNODE0}:/data +if (( $? != 0 )) +then + echo "Copy failed" + exit 1 +fi + +# copy restore data to ingest node +echo "Copying restore data from S3" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "aws s3 cp s3://molecula-perf-storage/able/perf-able-seg.tar.xz /data/perf-able-seg.tar.xz --no-progress" +if (( $? != 0 )) +then + echo "Copy failed" + exit 1 +fi + +# untar data +echo "Untarring data" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; tar -xf perf-able-seg.tar.xz" +if (( $? != 0 )) +then + echo "Untarring failed" + exit 1 +fi + +# restore data +echo "Restoring data" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "cd /data; featurebase restore --host http://${DATANODE0}:10101 -s /data/data/backup > restore.out" +if (( $? != 0 )) +then + echo "Restoring failed" + exit 1 +fi + +# run test +echo "Running perf test" +# leaving this here because K6 is timing out and need to work out why +#ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "/home/ec2-user/bin/k6 run -e DATANODE0=test.k6.io /data/highcardinalitygroupby.js" +ssh -A -i ~/.ssh/gitlab-featurebase-ci.pem -o "StrictHostKeyChecking no" ec2-user@${INGESTNODE0} "curl ${DATANODE0}:10101/index/seg/query -X POST -o /data/response.json -d 'GroupBy(Rows(education_level), Rows(gender), Rows(domain), aggregate=Sum(field=age))'" +ABLETESTRESULT=$? + +if (( $ABLETESTRESULT != 0 )) +then + echo "able perf test complete with failures" +else + echo "able test complete" +fi + +exit $ABLETESTRESULT \ No newline at end of file diff --git a/qa/scripts/perf/able/generateTestData.sh b/qa/scripts/perf/able/generateTestData.sh new file mode 100755 index 000000000..3193050fa --- /dev/null +++ b/qa/scripts/perf/able/generateTestData.sh @@ -0,0 +1,33 @@ +#!/bin/bash + + +# for --pilosa.hosts +PILOSA_HOSTS="" + +DEPLOYED_DATA_IPS=$(cat ./qa/tf/perf/able/outputs.json | jq -r '[.data_node_ips][0]["value"][]') +echo "DEPLOYED_DATA_IPS: {" +echo "${DEPLOYED_DATA_IPS}" +echo "}" + +DEPLOYED_DATA_IPS_LEN=`echo "$DEPLOYED_DATA_IPS" | wc -l` + +generatePilosaHostsString() { + IFS=$'\n' + cnt=0 + for ip in $DEPLOYED_DATA_IPS + do + if (($cnt + 1 != $DEPLOYED_DATA_IPS_LEN)) + then + PILOSA_HOSTS="${PILOSA_HOSTS}p${cnt}=$ip:10101," + else + PILOSA_HOSTS="${PILOSA_HOSTS}p${cnt}=$ip:10101" + fi + cnt=$((cnt+1)) + done + + echo "PILOSA_HOSTS: ${PILOSA_HOSTS}" +} + +generatePilosaHostsString + +datagen -s custom --custom-config=./able.yaml --pilosa.index=seg --pilosa.batch-size=1048576 --pilosa.hosts ${PILOSA_HOSTS} \ No newline at end of file diff --git a/qa/scripts/perf/able/highcardinalitygroupby.js b/qa/scripts/perf/able/highcardinalitygroupby.js new file mode 100644 index 000000000..da8e32d8f --- /dev/null +++ b/qa/scripts/perf/able/highcardinalitygroupby.js @@ -0,0 +1,11 @@ +import http from 'k6/http'; +import { sleep } from 'k6'; + +export default function () { + const params = { + timeout: '1800s', + }; + + let res = http.post(`http://${__ENV.DATANODE0}:10101/index/seg/query`, "GroupBy(Rows(education_level), Rows(gender), Rows(political_party), Rows(domain), aggregate=Sum(field=age))", params); + sleep(1); +} \ No newline at end of file diff --git a/qa/scripts/perf/able/values/education.txt b/qa/scripts/perf/able/values/education.txt new file mode 100644 index 000000000..4e7c28b29 --- /dev/null +++ b/qa/scripts/perf/able/values/education.txt @@ -0,0 +1,6 @@ +Some High School +High School +Some College +College +Master's +Doctorate \ No newline at end of file diff --git a/qa/scripts/perf/able/values/gender.txt b/qa/scripts/perf/able/values/gender.txt new file mode 100644 index 000000000..4ae957346 --- /dev/null +++ b/qa/scripts/perf/able/values/gender.txt @@ -0,0 +1,5 @@ +Male,0.48 +Female,0.48 +Transgender,0.01 +Other,0.01 +Unspecified,0.02 \ No newline at end of file diff --git a/qa/scripts/perf/able/values/hobbies.txt b/qa/scripts/perf/able/values/hobbies.txt new file mode 100644 index 000000000..3da9bdee2 --- /dev/null +++ b/qa/scripts/perf/able/values/hobbies.txt @@ -0,0 +1,642 @@ +Lego building +Watching movies +Watch making +Slacklining +BMX +Cricket +Sketching +Satellite watching +Volunteering +Radio-controlled model playing +Stone collecting +Picnicking +Hydroponics +Karate +Roller skating +Skateboarding +Element collecting +Weaving +Beach volleyball +Archery +Livestreaming +Stone skipping +Trapshooting +Filmmaking +Diorama +Makeup +Rugby league football +Community activism +Field hockey +Backpacking +Slot car +Insect collecting +VR Gaming +Video making +Bowling +Sled dog racing +Skiing +Web design +Sand art +Public speaking +Movie memorabilia collecting +Gardening +Wikipedia editing +Croquet +Mathematics +Rail transport modeling +Darts +Judo +Equestrianism +Figure Skating +Scrapbooking +Airbrushing +Photography +Climbing +Tourism +Journaling +Flower growing +Wood carving +Fashion design +Polo +Slot car racing +Reading +Electronic games +Martial arts +Bell ringing + Air sports +Skipping rope +Bowling +Caving +Leather crafting +Construction +Bus riding +Flag football +Anime +Whittling +Aerospace +Sun bathing +Music +Running +Diving +Plastic art +Stamp collecting +Gymnastics +Kabaddi +Coin collecting +Video editing +Stripping +Cribbage +Candy making +Amateur geology +Motor sports +Sculpting +Transit map collecting +Refinishing +Surfing +Swimming +Skateboarding +Knowledge/word games +Tether car +Poi +Manga + Action figure +Teaching +Blacksmithing +Fingerpainting +Audiophile +Spreadsheets +Scouting +Frisbee +Metal detecting +Book collecting +Radio-controlled model playing +Films +Karaoke +Wargaming +Biology +DJing +Axe throwing +Volleyball +Life Science +Fossil hunting +Beachcombing +Sudoku +Cross-stitch +Ephemera collecting +Puzzles +Hiking/backpacking +Digital hoarding +Horseshoes +Amateur astronomy +Book discussion clubs +Model building +Ceramics +Telling jokes +Gardening +Renaissance fair +Record collecting +Collecting +Taxidermy +Flying +Zumba + Archaeology +Quidditch +Playing musical instruments +Tapestry +Perfume +Philately +Business +Microbiology +Rafting +Postcrossing +Whisky +Botany +Badminton +Chatting +Board sports +Groundhopping +Inventing +Paragliding +Shooting sport +Esports +Sport stacking +Proverbs +Marching band +Feng shui decorating +Car tuning +Sociology +Writing music +Robot combat +Parkour +Shogi +Weightlifting +Fashion +Safari +Motorcycling +Pool +Meteorology +Auto audiophilia +Mushroom hunting/mycology +Radio-controlled model playing +Miniature art +Video game developing +Medical science +Herp keeping +Shoemaking +Gongfu tea +Dowsing +Microscopy +Welding +Woodworking +Clothesmaking +Fingerprint collecting +Crossword puzzles +Breadmaking +Ice hockey +Dolls +Curling +Sailing +Mazes (indoor/outdoor) +Fishkeeping +Ticket collecting +Flower arranging +Nail art +Couponing +Skimboarding +Fishing +Figure skating +Herping +Surfing +Go +Vintage clothing +Shortwave listening +Water sports +Darts +Bonsai +Lomography +Crocheting +Meditation +Cornhole +Railway journeys +Cardistry +Book restoration +Graffiti +Decorating +Yo-yoing +Speedcubing +Lotology (lottery ticket collecting) +Houseplant care +Cryptography +Quilling +Powerlifting +Cheesemaking +Table tennis +Public transport riding +Pet adoption & fostering +Magnet fishing +Hooping +Bridge +Rubik's Cube +Beekeeping +Digital arts +Foreign language learning +Race walking +Fusilately (phonecard collecting) +Fishfarming +Jigsaw puzzles +Reviewing Gadgets +Entrepreneurship +Pickleball +Wine tasting +Footbag +Astronomy +Stuffed toy collecting +Roller derby +Astrology +Furniture building +Lapidary +Iceboat racing +High-power rocketry +Reiki +Baking +Automobilism +Witchcraft +Walking +Aerial silk +Gongoozling +Learning +Cartophily (card collecting) +Paintball +Genealogy +Do it yourself +Volleyball +Science and technology studies +Horsemanship +Swimming +Needlepoint +Fishkeeping +Vintage cars +Basketball +Qigong +Video game collecting +Writing +Vacation +Nordic skating +Powerboat racing +Baseball +Candle making +Whale watching +Knot tying +Ice skating +Debate +Checkers (draughts) +Board/tabletop games +Model engineering +VR Gaming +Palmistry +Air hockey +Pole dancing +Modeling +Puppetry +Memory training +Sculling or rowing +Seashell collecting +Poetry +Role-playing games +Flying model planes +Tennis polo +Gymnastics +Metalworking +Scutelliphily +Eating +Pet sitting +Fruit picking +Farming +Survivalism +Fly tying +Wax sealing +Sea glass collecting +Antiquing +Metal detecting +Guerrilla gardening +Dance +Birdwatching +Skiing +Jujitsu +Hiking +Model aircraft +Model United Nations +Jukskei +Leaves +Drama +Lacrosse +LARPing +Home improvement +Skydiving +Snowmobiling +Meteorology +Fantasy sports +Blogging +Hobby horsing +Knife throwing +English +Soapmaking +Talking +Lace making +Driving +Engraving +Kung fu +Laser tag +Composting +Sledding +Croquet +Railway studies +Magic +Kite flying +Acting +Juggling +Travel +Glassblowing +Baton twirling +Boxing +Kart racing +Comic book collecting +Meditation +Mineral collecting +Dancing +Antiquities +Ultimate frisbee +Planning +Pole dancing +Snorkeling +Zoo visiting +Animation +Rock painting +Exhibition drill +Stamp collecting +People-watching +Knife collecting +Herbalism +Knitting +Karting +Tennis +Drink mixing +Kombucha brewing +Chemistry +Badminton +Lock picking +Letterboxing +Storm chasing +Sports memorabilia +Tai chi +Calligraphy +Weight training +Pin (lapel) +Coffee roasting +Unicycling +Ghost hunting +Archery +Museum visiting +Card games +Dog sport +Herping +Netball +Video gaming +Trade fair visiting +Baseball +plush collecting +Car fixing & building +Tatebanko +BASE jumping +Gold prospecting +Animal fancy +Jogging +Gunsmithing +Shooting +Long-distance running +Quizzes +Canoeing +Aquascaping +Practical jokes +Tattooing +Social studies +Vehicle restoration +Cheerleading +Proofreading and editing +Fishing +Squash +Tarot +Sewing +Birdwatching +Cycling +Button collecting +Animation +Art +Giving advice +Handball +Die-cast toy +Jewelry making +Deltiology (postcard collecting) +Brazilian jiu-jitsu +Coloring +Podcast hosting +Couch surfing +Reading +Compact discs +Bullet journaling +Hunting +Australian rules football +Origami +Tea bag collecting +Webtooning +Longboarding +Auto detailing +Hacking +Kendama +Photography +Pilates +Snowboarding +Pressed flower craft +Conlanging +Beatboxing +Amateur radio +Freestyle football +Mountaineering +Rock tumbling +Yoga +Bus spotting +Tour skating +Rock balancing +Camping +Sculling or rowing +Performance +Djembe +Entertaining +Chess +Cleaning +Electronics +Vinyl Records +Beauty pageants +Auto racing +Climbing +Road biking +Gingerbread house making +Distro Hopping +Geocaching +Snowshoeing +Creative writing +Taekwondo +Radio-controlled car racing +Worldbuilding +Car riding +Stand-up comedy +Flying disc +Dog walking +Phillumeny +Foraging +Singing +Barbershop Music +Confectionery +Amusement park visiting +Inline skating +Knife making +History +Breakdancing +Experimenting +Color guard +Painting +Soccer +Backgammon +City trip +Marbles +Renovating +Speed skating +Handball +Gaming +Triathlon +Mountain biking +Machining +Art collecting +Baton twirling +Horseback riding +Benchmarking +Philately +Tourism +Wrestling +Disc golf +Flower collecting and pressing +Fitness +Acroyoga +Beer tasting +Video gaming +Lacrosse +Bodybuilding +Thrifting +Topiary +3D printing +Crystals +Orienteering +Noodling +Geocaching +Orienteering +Winemaking +Watching documentaries +Pet +Drawing +Photography +Airsoft +Homebrewing +Aircraft spotting +Mini Golf +Storytelling +Pickleball +Shuffleboard +Cooking +Rock climbing +Vegetable farming +Radio-controlled model playing +Billiards +Association football +Embroidery +Waxing +Physics +Hobby tunneling +Scuba diving +Kayaking +Videography +Tennis +Slot car +Table tennis +Golfing +Dog training +Craft +Mahjong +Cycling +Thru-hiking +Fencing +Airsoft +Humor +Mycology +Rail transport modelling +Sports science +Table football +Trainspotting +Minimalism +Urban exploration +Macrame +Computer programming +Horseback riding +Cue sports +Magic +Pyrography +Ice skating +Upcycling +Shoes +Power Nap +Pen Spinning +Jumping rope +Astronomy +Pottery +Martial arts +Butterfly watching +Hula hooping +Water polo +Geography +Chess +Rugby +Cosplaying +Racquetball +Shopping +Graphic design +Binge-watching +Kitesurfing +Research +Model racing +Listening to podcasts +Radio-controlled model collecting +Research +Rapping +Poker +Rappelling +Watching television +Listening to music +Mechanics +Philosophy +Recipe creation +Quilting +Fossicking +Social media +Word searches +Massaging +Dominoes +Longboarding +Scuba Diving +Dining +Hardware +Communication +Ant-keeping +Canyoning +Dandyism +Psychology +Softball +Table tennis playing diff --git a/qa/scripts/perf/able/values/income.txt b/qa/scripts/perf/able/values/income.txt new file mode 100644 index 000000000..c7149440d --- /dev/null +++ b/qa/scripts/perf/able/values/income.txt @@ -0,0 +1,7 @@ +$0-$14,200 +$14,201-$54,200 +$54,201-$86,350 +$86,351-$164,900 +$164,901-$209,400 +$209,401-$523,600 +$523,601 or more \ No newline at end of file diff --git a/qa/scripts/perf/able/values/opendns-top-domains-10K.txt b/qa/scripts/perf/able/values/opendns-top-domains-10K.txt new file mode 100644 index 000000000..23ca2c898 --- /dev/null +++ b/qa/scripts/perf/able/values/opendns-top-domains-10K.txt @@ -0,0 +1,10000 @@ +google.com +facebook.com +doubleclick.net +google-analytics.com +akamaihd.net +googlesyndication.com +googleapis.com +googleadservices.com +facebook.net +youtube.com +twitter.com +scorecardresearch.com +microsoft.com +ytimg.com +googleusercontent.com +apple.com +msftncsi.com +2mdn.net +googletagservices.com +adnxs.com +yahoo.com +serving-sys.com +akadns.net +bluekai.com +ggpht.com +rubiconproject.com +verisign.com +addthis.com +crashlytics.com +amazonaws.com +quantserve.com +akamaiedge.net +live.com +googletagmanager.com +revsci.net +adadvisor.net +openx.net +digicert.com +pubmatic.com +agkn.com +instagram.com +mathtag.com +gmail.com +rlcdn.com +linkedin.com +yahooapis.com +chartbeat.net +twimg.com +turn.com +crwdcntrl.net +demdex.net +betrad.com +flurry.com +newrelic.com +yimg.com +youtube-nocookie.com +exelator.com +acxiom-online.com +imrworldwide.com +amazon.com +fbcdn.net +windowsupdate.com +mookie1.com +rfihub.com +omniroot.com +adsrvr.org +nexac.com +bing.com +skype.com +godaddy.com +sitescout.com +tubemogul.com +contextweb.com +w55c.net +chartbeat.com +akamai.net +jquery.com +adap.tv +criteo.com +krxd.net +optimizely.com +macromedia.com +comodoca.com +casalemedia.com +pinterest.com +adsymptotic.com +symcd.com +atwola.com +adobe.com +msn.com +adsafeprotected.com +tapad.com +truste.com +symantecliveupdate.com +atdmt.com +t.co +avast.com +google.co.in +spotxchange.com +tidaltv.com +adtechus.com +everesttech.net +addthisedge.com +hola.org +btrll.com +gwallet.com +liverail.com +windows.com +burstnet.com +disqus.com +nr-data.net +p-td.com +geotrust.com +admob.com +crittercism.com +bizographics.com +ru4.com +wtp101.com +ksmobile.com +msads.net +thawte.com +lijit.com +cloudflare.com +360yield.com +dropbox.com +simpli.fi +smartadserver.com +globalsign.com +mlnadvertising.com +chango.com +connexity.net +moatads.com +s-msn.com +entrust.net +tribalfusion.com +domdex.com +google.com.tr +whatsapp.net +ntp.org +amazon-adsystem.com +viber.com +disquscdn.com +yandex.ru +doubleverify.com +bkrtx.com +criteo.net +outbrain.com +questionmarket.com +adform.net +yieldmanager.com +typekit.net +goo.gl +voicefive.com +owneriq.net +media6degrees.com +tynt.com +symcb.com +advertising.com +audienceiq.com +wp.com +rtbidder.net +wikipedia.org +adroll.com +icloud.com +gravatar.com +collective-media.net +appsflyer.com +dmtry.com +blogger.com +taboola.com +legolas-media.com +images-amazon.com +afy11.net +aspnetcdn.com +hike.in +feedburner.com +bootstrapcdn.com +usertrust.com +adgrx.com +brilig.com +sharethis.com +flashtalking.com +mediaplex.com +eqads.com +adscale.de +imgur.com +edgesuite.net +blogspot.com +msocsp.com +wikimedia.org +ssl-images-amazon.com +amung.us +flickr.com +rundsp.com +trouter.io +edgekey.net +rfihub.net +utorrent.com +thebrighttag.com +eyeviewads.com +switchads.com +tiqcdn.com +mozilla.org +jwpcdn.com +exponential.com +abmr.net +nanigans.com +zenoviaexchange.com +aolcdn.com +licdn.com +mixpanel.com +254a.com +mopub.com +creative-serving.com +statcounter.com +jwpltx.com +parse.com +ensighten.com +adtech.de +brightcove.com +acuityplatform.com +gfx.ms +ixiaa.com +reddit.com +visualrevenue.com +google.com.br +stickyadstv.com +google.it +yashi.com +jumptap.com +interclick.com +tapjoyads.com +globalsign.net +eyereturn.com +pointroll.com +googlevideo.com +virtualearth.net +gumgum.com +triggit.com +tumblr.com +gigya.com +teamviewer.com +insightexpressai.com +msecnd.net +gemius.pl +oracle.com +sonobi.com +fastclick.net +ebay.com +adobetag.com +surveymonkey.com +stumbleupon.com +admaym.com +invitemedia.com +superfish.com +google.com.vn +yahoodns.net +tapjoy.com +blogblog.com +mxpnl.com +omtrdc.net +skimresources.com +akamai.com +adobedtm.com +starfieldtech.com +skypeassets.com +a.com +btstatic.com +researchnow.com +conviva.com +hotmail.com +bittorrent.com +openbittorrent.com +vindicosuite.com +duba.net +publicbt.com +impact-ad.jp +netflix.com +ib-ibi.com +smaato.net +netsolssl.com +fetchback.com +appspot.com +vk.com +mozilla.com +accu-weather.com +yieldmanager.net +yadro.ru +histats.com +netseer.com +creativecommons.org +live.net +vizu.com +youtu.be +kau.li +eyeota.net +weather.com +provenpixel.com +veruta.com +umengcloud.com +paypal.com +office365.com +simplereach.com +ooyala.com +specificclick.net +digg.com +google.ca +dotomi.com +netmng.com +undertone.com +erne.co +staticflickr.com +urbanairship.com +adkmob.com +pro-market.net +dtscout.com +imdb.com +mzstatic.com +alexa.com +fastly.net +baidu.com +brealtime.com +amazon.co.uk +midasplayer.com +bugsense.com +outlook.com +chartboost.com +adrta.com +adcash.com +root-servers.net +adtilt.com +awstls.com +fwmrm.net +cdninstagram.com +adsonar.com +zedo.com +demonii.com +vimeo.com +dianxinos.com +adventori.com +accuweather.com +steamstatic.com +coull.com +mxptint.net +pfx.ms +footprint.net +ceipmsn.com +paypalobjects.com +taboolasyndication.com +umeng.com +altitude-arena.com +webtrendslive.com +dl-rms.com +visualwebsiteoptimizer.com +mydas.mobi +cap-mii.net +naver.jp +avg.com +wordpress.com +pinimg.com +livefyre.com +tabwpm.us +maxymiser.net +wordpress.org +ebayimg.com +gravity.com +huffingtonpost.com +exoclick.com +pandora.com +reson8.com +grvcdn.com +aol.com +adcolony.com +adhigh.net +eset.com +trustwave.com +cnn.com +cxense.com +lfstmedia.com +xboxlive.com +vungle.com +a3cloud.net +dailymotion.com +postrelease.com +duapp.com +king.com +mailshell.net +pingdom.net +lenovomm.com +dyntrk.com +kaspersky-labs.com +jwpsrv.com +nsatc.net +soundcloud.com +vimeocdn.com +theviilage.com +hlserve.com +wdgserv.com +inmobi.com +bbc.co.uk +kaspersky.com +spotxcdn.com +norton.com +nytimes.com +crsspxl.com +liveperson.net +amgdgt.com +amazon.in +amazon.de +adotube.com +go.com +samsungosp.com +parsely.com +windowsphone.com +heias.com +amazon.it +washingtonpost.com +ospserver.net +mscimg.com +google.co.uk +mzl.la +pswec.com +media.net +v0cdn.net +supercell.net +visadd.com +andomedia.com +mdotlabs.com +adformdsp.net +wikimediafoundation.org +alenty.com +zergnet.com +sundaysky.com +amazon.ca +mediawiki.org +datafastguru.info +vidible.tv +adzerk.net +brand-server.com +quantcount.com +flipboard.com +dtmpub.com +spongecell.com +tinyurl.com +clkmon.com +bing.net +adlegend.com +adblockplus.org +dvtps.com +p-cdn.com +mailchimp.com +wikidata.org +icio.us +ebaystatic.com +viglink.com +ibook.info +itools.info +thinkdifferent.us +airport.us +appleiphonecell.com +hwcdnlb.net +effectivemeasure.net +amazon.fr +iponweb.net +mbamupdates.com +foxnews.com +fiksu.com +dlqm.net +ozonemedia.com +zenfs.com +deliads.com +yieldlab.net +sail-horizon.com +applovin.com +nspmotion.com +metrigo.com +pulsemgr.com +visiblemeasures.com +revenuemantra.com +smartclip.net +ijinshan.com +tndmnsha.com +go-mpulse.net +relestar.com +amazon.co.jp +jollywallet.com +trafficmanager.net +imgfarm.com +opera-mini.net +cogocast.net +onenote.com +amazon.es +opendns.com +p161.net +a-msedge.net +cpmstar.com +amazon.com.br +logmein.com +nflximg.net +univide.com +tekblue.net +infostatsvc.com +udmserve.net +basebanner.com +zynga.com +amazon.cn +mathads.com +amazon.com.au +mediade.sk +atemda.com +d41.co +amazon.com.mx +airpush.com +ksmobile.net +geogslb.com +goodreads.com +monetate.net +clicktale.net +richrelevance.com +tns-counter.ru +coremetrics.com +online-metrix.net +rs6.net +xingcloud.com +generalmobi.com +uservoice.com +herokuapp.com +adblade.com +svcmot.com +shopbop.com +z5x.net +optmd.com +dropboxusercontent.com +fbsbx.com +turner.com +onclickads.net +bookdepository.com +bluecava.com +adtimaserver.vn +beringmedia.com +choicestream.com +zanox.com +apsalar.com +realmedia.com +dpclk.com +cedexis.com +scanscout.com +display-trk.com +bitmedianetwork.com +ctnsnet.com +tunigo.com +samsung.com +bazaarvoice.com +ebayrtm.com +returnpath.net +walmart.com +wsod.com +constantcontact.com +getclicky.com +localytics.com +ligatus.com +appier.net +dxsvr.com +myhabit.com +ajaxcdn.org +adyapper.com +nist.gov +neulion.com +edgecastcdn.net +convertro.com +vnexpress.net +javafx.com +thepiratebay.org +skype.net +kontagent.net +newsinc.com +glpals.com +ebz.io +audible.com +mobogenie.com +dingaling.ca +nrcdn.com +stumble-upon.com +backupgrid.net +po.st +marinsm.com +nflximg.com +adizio.com +acx.com +fyre.co +admedo.com +xvideos.com +junglee.com +evernote.com +createspace.com +buzzfeed.com +zing.vn +sanasecurity.com +igexin.com +bnmla.com +liadm.com +usatoday.com +scanalert.com +espncdn.com +metamx.com +plexop.net +optimatic.com +medyanetads.com +w3.org +apnanalytics.com +gezinti.com +dpreview.com +xbox.com +servesharp.net +cpxinteractive.com +adsparc.net +cardlytics.com +dailymail.co.uk +redditstatic.com +sociomantic.com +contentabc.com +admost.com +inmobicdn.net +3g.cn +miisolutions.net +nrelate.com +innovid.com +nola.com +testflightapp.com +teads.tv +fool.com +tripadvisor.com +al.com +cloudapp.net +public-trust.com +vine.co +mlive.com +cleveland.com +tp-cdn.com +addtoany.com +sharethrough.com +clickfuse.com +nj.com +abebooks.com +batanga.net +mediavoice.com +wsodcdn.com +bloomberg.com +ucweb.com +fonts.com +videohub.tv +spotify.com +alicdn.com +cdngc.net +groupon.com +afterschool.com +symantec.com +oregonlive.com +apptimize.com +trafficfactory.biz +ibillboard.com +vizury.com +qservz.com +perfectmarket.com +yieldoptimizer.com +ad4game.com +ask.com +networkhm.com +amazonlocal.com +zappos.com +diapers.com +adtricity.com +ml314.com +yldbt.com +plexop.com +bbb.org +tworismo.com +amazonsupply.com +beautybar.com +theguardian.com +myhomemsn.com +nvidia.com +comixology.com +bookworm.com +huffpost.com +vcmedia.vn +casa.com +woot.com +eastdane.com +answers.com +infolinks.com +fabric.com +lphbs.com +rpxnow.com +ovi.com +dlinksearch.com +adlooxtracking.com +soap.com +mail.ru +look.com +microsoftonline.com +wag.com +dyndns.org +pennlive.com +nbcnews.com +yoyo.com +zopim.com +collserve.com +vine.com +gpsonextra.net +tacoda.net +trusteer.com +yahoo.net +toolbarservices.com +bluelithium.com +sun.com +33across.com +ipinfo.io +iasds01.com +longtailvideo.com +typography.com +6pm.com +ptvcdn.net +adf.ly +kissmetrics.com +ccc.de +c3tag.com +safemovedm.com +tango.me +bbc.com +syracuse.com +dashbida.com +gvt1.com +admicro.vn +sascdn.com +r1-cdn.net +everestjs.net +craigslist.org +llnwd.net +thanksearch.com +iegallery.com +typekit.com +visualdna.com +angsrvr.com +tenmarks.com +mediaforge.com +telegraph.co.uk +myspace.com +lastpass.com +steampowered.com +startssl.com +ipinyou.com +fonts.net +goo.mx +google.com.mx +tr553.com +5min.com +tfxiq.com +korrelate.net +alibaba.com +mininova.org +ebaydesc.com +desync.com +compete.com +kochava.com +kaltura.com +bleacherreport.com +buscape.com.br +flite.com +swisssign.net +yieldmo.com +content.ad +github.com +wsj.com +opera.com +grouponcdn.com +aliunicorn.com +solocpm.com +nav-links.com +crtinv.com +hiro.tv +opendsp.com +windows.net +dmcdn.net +wii.com +farlex.com +smartstream.tv +yandex.net +masslive.com +blogher.org +jccjd.com +beanstock.co +weatherbug.com +intellitxt.com +bidtheatre.com +mmondi.com +linkedinlabs.com +acrobat.com +nokia.com +levexis.com +cbsi.com +adsplats.com +perfectaudience.com +admarvel.com +performgroup.com +liveinternet.ru +zyngawithfriends.com +bankrate.com +24h.com.vn +trafficjunky.net +cedexis.net +janrain.com +geforce.com +tacdn.com +eonline.com +smarturl.it +impdesk.com +internapcdn.net +umeng.co +sekindo.com +steamcommunity.com +riotgames.com +wunderground.com +nextadvisor.com +reuters.com +vibrant.co +blackberry.com +hwcdn.net +tremormedia.com +netgear.com +fncstatic.com +google.com.eg +ebdr3.com +revcontent.com +businessinsider.com +prfct.co +iperceptions.com +c8.net.ua +taobao.com +delicious.com +247realmedia.com +imwx.com +active-agent.com +supersonicads.com +realtime.co +kill123.com +phncdn.com +redditmedia.com +thepostgame.com +h33t.com +a9.com +foursquare.com +milliyet.com.tr +4dsply.com +upwpm.us +csze.com +mediaquark.com +tritondigital.com +mozilla.net +fidelity-media.com +dmca.com +greystripe.com +cafemom.com +mapticket.net +xhamster.com +ow.ly +maxmind.com +avira.com +webspectator.com +marketo.net +vlingo.com +iesnare.com +qwapi.com +rarbg.com +twitch.tv +myfonts.net +aws-protocol-testing.com +cb-cdn.com +segment.io +adnetwork.vn +qq.com +kik.com +technoratimedia.com +res-x.com +samsungapps.com +lenovo.com +americanexpress.com +htc.com +android.com +apnstatic.com +bounceexchange.com +tumri.net +theplatform.com +olark.com +cnbc.com +thespatialists.com +shareaholic.com +specificmedia.com +sharedaddomain.com +jquerytools.org +microadinc.com +clashofclans.com +roku.com +qualtrics.com +thescene.com +medialytics.com +mashable.com +cubecdn.net +360game.vn +estara.com +kiip.me +aliexpress.com +dailyofferservice.com +uol.com.br +adk2.co +aliimg.com +tentaculos.net +jsuol.com +attracto.com +corom.vn +dessaly.com +sgiggle.com +mobileapptracking.com +office.com +linkwithin.com +latimes.com +cbsnews.com +eclick.vn +glbimg.com +epicunitscan.info +avira-update.com +hoptopboy.com +tvlsvc.com +tailtarget.com +desk.com +intentiq.com +ero-advertising.com +imguol.com +everyscreenmedia.com +bbci.co.uk +itunes.com +engadget.com +people.com +dsply.com +voga360.com +hmageo.com +337play.com +gannett-cdn.com +rcsadv.it +manage.com +cachefly.net +doublepimp.com +keen.io +ea.com +reklamport.com +shopping.com +youradexchange.com +hp.com +apptentive.com +earthnetworks.com +nfl.com +userdmp.com +yastatic.net +google.de +apxlv.com +moneynews.com +livechatinc.com +forbes.com +pornhub.com +sbal4kp.com +wsoddata.com +logmein-gateway.com +facdn.com +yldmgrimg.net +hurriyet.com.tr +lucidmedia.com +doracdn.com +indeed.com +disneytermsofuse.com +truecaller.com +time.com +mediatek.com +ioam.de +rackcdn.com +baidu.co.th +reklamstore.com +pricegrabber.com +dyndns.com +imageshack.us +popads.net +dataxu.com +sndcdn.com +gizmodo.com +imageshack.com +yelp.com +google.ru +best-tv.com +webtrends.com +google.fr +archive.org +walmartimages.com +att.com +e-planning.net +openxenterprise.com +yan.vn +company-target.com +cmptch.com +incmd04.com +disneyprivacycenter.com +npr.org +tellapart.com +hulu.com +dynamicyield.com +theatlantic.com +atgsvcs.com +whois.co.kr +life360.com +tmz.com +visualstudio.com +adservingml.com +securetrust.com +qubitproducts.com +360.cn +realvu.net +fortune.com +sitescoutadserver.com +sponsorpay.com +torrentum.pl +brcdn.com +origin.com +slidesharecdn.com +360safe.com +pressroomvip.com +unrulymedia.com +nxtck.com +adexcite.com +etsy.com +odnoklassniki.ru +iheart.com +mmstat.com +glam.com +radaronline.com +popnhop.com +edgefcs.net +redintelligence.net +myvisualiq.net +mgid.com +2o7.net +mapquest.com +mediamath.com +me.com +ugdturner.com +amasvc.com +monster.com +seethisinaction.com +ebayinc.com +wallstcheatsheet.com +sogou.com +ambient-platform.com +traffichaus.com +kinja-img.com +googlecommerce.com +utorrent.li +thoiloan.vn +dantri.com.vn +ubuntu.com +googlecode.com +google.com.ar +coppersurfer.tk +garenanow.com +flx1.com +1337x.org +videosz.com +virool.com +kenh14.vn +nypost.com +octro.net +ztstatic.com +stackoverflow.com +wishabi.com +jsdelivr.net +vitrines.in +media-imdb.com +predicta.net +cmcore.com +appoxee.com +mcafeesecure.com +crowdscience.com +pagefair.com +adlucent.com +chase.com +nydailynews.com +padsdelivery.com +wlxrs.com +adscience.nl +shoppingshadow.com +mradx.net +fotapro.com +wired.com +cdn.md +hubspot.com +google.es +buzzfed.com +comcast.net +polldaddy.com +plexapp.com +hidemyass.com +steelhousemedia.com +yumenetworks.com +acc-hd.de +populisengage.com +bncnt.com +responsys.net +printfriendly.com +zendesk.com +gmtdmp.com +madisonlogic.com +dartsearch.net +zdn.vn +zedo.net +nbcudigitaladops.com +stubhub.com +adhood.com +microsofttranslator.com +espn.com +linksmart.com +wshifen.com +appa-maker.com +cabelas.com +redtube.com +channelintelligence.com +dell.com +weibo.com +channeladvisor.com +viewster.com +adjuggler.net +xnxx.com +adxpansion.com +alibench.com +qadservice.com +mybuys.com +raasnet.com +tanx.com +popmarker.com +pubnub.com +peer39.net +globo.com +weborama.fr +independent.co.uk +searchmarketing.com +zemanta.com +vgtf.net +inspsearchapi.com +rambler.ru +en25.com +gomonetworks.com +playhaven.com +aweber.com +retargetly.com +allvoices.com +intel.com +pubsqrd.com +admized.com +minimob.com +adingo.jp +cnet.com +userreport.com +trustedsource.org +vk.me +mediafire.com +buysellads.com +slideshare.net +sexad.net +windowsmedia.com +tremorhub.com +licasd.com +bycontext.com +echoenabled.com +issuu.com +1mobile.com +corporate-ir.net +pubexchange.com +audienceinsights.net +adobur.com +celtra.com +techcrunch.com +boo-box.com +eum-appdynamics.com +try9.com +adriver.ru +taobaocdn.com +dealtime.com +ed4.net +trust-provider.com +feedbackify.com +bbelements.com +dwin1.com +yandex.st +gssp-a.com +4seeresults.com +adition.com +nhncorp.jp +googlemail.com +about.com +gap.com +hotwords.com.br +ant.com +plugrush.com +foreseeresults.com +bidswitch.net +gawker.com +advidi.com +pagefair.net +mixpo.com +intuit.com +imiclk.com +bestbuy.com +engageya.com +nexage.com +intergi.com +playstation.net +foxbusiness.com +adk2.com +9999mb.com +bitdefender.net +cpserve.com +yb0t.com +mi-idc.com +espn.co.uk +minecraft.net +crossrider.com +conduit.com +sensic.net +pavv.co.kr +telemetryverification.net +metanetwork.net +lifehacker.com +bbcimg.co.uk +today.com +jtvnw.net +ptreklam.com.tr +inspsearch.com +poll.fm +komoona.com +v2cdn.net +adtima.vn +viralnova.com +harry.lu +trialpay.com +m6r.eu +samsungrm.net +vindicosuitecache.com +rarbg.me +pusherapp.com +asus.com +indexww.com +assoc-amazon.com +ask.fm +yandex.com.tr +adpredictive.com +swiftkey.net +csdata1.com +kontera.com +reddit.tv +baidustatic.com +ctmail.com +gotinder.com +siteadvisor.com +applifier.com +gtimg.com +crdrdpjs.info +redditgifts.com +boldchat.com +dataxu.net +wishabi.net +dynad.net +legacy.com +emjcd.com +cbsimg.net +google.com.hk +pop6.com +t-mobile.com +anthill.vn +zdbb.net +sitewebred.info +youporn.com +radiumone.com +whatsapp.com +technorati.com +aim.net +dotandad.com +ex.ua +adsrvmedia.net +lineage2.com.cn +metaffiliation.com +mywot.com +ns-img.com +shoplocal.com +cloudinary.com +creativecdn.com +vdna-assets.com +doi.org +newsmaxfeednetwork.com +rantlifestyle.com +thedailybeast.com +adjuggler.com +huffpo.net +shopify.com +bitly.com +trtromg.com +samsungotn.net +ups.com +hlntv.com +spccint.com +domobile.com +shinystat.com +worldssl.net +infospace.com +chtah.com +vaporcloudcomputing.com +firstimpwins.com +factual.com +ad360.vn +nmcdn.us +adgear.com +theverge.com +mapquestapi.com +comodoca2.com +scdn.co +sstatic.net +kgridhub.com +coccoc.com +businessweek.com +etonline.com +olx.com +eepurl.com +inspectlet.com +marketwatch.com +rklyjs.info +googledrive.com +ford.com +ants.vn +comufy.com +adshost1.com +ns-cdn.com +q1mediahydraplatform.com +tmall.com +booking.com +fivethirtyeight.com +juicyads.com +groovinads.com +plug.it +myvzw.com +semasio.net +nih.gov +cbsinteractive.com +gandi.net +appclick.co +githubusercontent.com +gogorithm.com +openweathermap.org +directrev.com +pow7.com +io9.com +ok.ru +cdnads.com +updatepm.com +chitika.net +vnecdn.net +sailthru.com +fb.me +zencdn.net +salon.com +espnfc.us +mouseflow.com +mainadv.com +healthcentral.com +novanet.vn +aarp.org +wistia.net +moneymorning.com +yceml.net +netdna-cdn.com +moviefone.com +gittigidiyor.com +adbrn.com +sahibinden.com +java.com +videoplaza.tv +videoamp.com +secureserver.net +kinja-static.com +padstm.com +nocookie.net +timeinc.net +webmd.com +xg4ken.com +haberturk.com +radioreddit.com +trovi.com +hs-analytics.net +estadao.com.br +bankofamerica.com +noproblemppc.com +hollywoodreporter.com +ad-score.com +newinfoclientstack.com +somo.vn +swrve.com +accmgr.com +civicscience.com +ft.com +worldnow.com +charter.com +polyad.net +si.com +webengage.com +mobfox.com +google.nl +millennialmedia.com +dataferb.com +vkontakte.ru +ff0000-cdn.net +billboard.com +beanstock.com +mochibot.com +wiktionary.org +cnn.co.jp +blankbase.com +fedex.com +ywxi.net +sitemeter.com +ap.org +vitrinesglobo.com.br +admission.net +unity3d.com +zedge.net +hackerwatch.org +gameanalytics.com +wistia.com +petuniasaucecockup.com +whaleserver.com +glympse.com +nintendo.net +cbssports.com +mplxtms.com +recaptcha.net +qlogo.cn +tube8.com +speedtest.net +webtrekk.com +ngoisao.net +juiceadv.com +datropy.com +kinja.com +inc.com +office.net +everestads.net +securespy.net +optorb.com +google.dz +mobify.com +sony.net +intellicast.com +sbnation.com +sourceforge.net +stackexchange.com +thehill.com +mindspark.com +telecomitalia.it +iobit.com +slimspots.com +haberler.com +espncms.com +newyorker.com +myinfotopia.com +adsrv247.com +rtalabel.org +espnfc.com +solvemedia.com +espncareers.com +fcc.gov +3lift.com +neodatagroup.com +sitebeacon.co +snapwidget.com +timeinc.com +pardot.com +admarketplace.net +usmagazine.com +admeld.com +pcfaster.com +adinterax.com +adlure.net +mqcdn.com +gm.com +itim.vn +loading-delivery1.com +usabilla.com +janrainbackplane.com +nbcsports.com +chatango.com +affec.tv +tlvmedia.com +integral.com +wealthfront.com +dsrlte.com +kohls.com +belkin.com +rdrtr.com +careerbuilder.com +leagueoflegends.com +eamobile.com +circularhub.com +linksynergy.com +irs01.com +bannerflow.com +lifestylejournal.com +dickssportinggoods.com +cnnexpansion.com +token.ro +bizrate.com +tfbnw.net +etsystatic.com +answcdn.com +cnnimagesource.com +vox-cdn.com +innity.net +nyt.com +powerreviews.com +adfox.vn +cnnchile.com +helpshift.com +parastorage.com +itau.com.br +9gag.com +appsdt.com +netvibes.com +stellaservice.com +afamily.vn +connextra.com +nbcuni.com +4wnet.com +dedicatedmedia.com +no-ip.com +espnmediazone.com +luminate.com +slate.com +openstreetmap.org +lazada.vn +sophosupd.com +free-coupons-codes.com +wfxtriggers.com +grantland.com +struq.com +latinsoulstudio.com +mixplay.tv +lomadee.com +ypcdn.com +alibabagroup.com +target.com +linknavi1.com +anyclip.com +woopra.com +pg.com +kickass.to +scribd.com +aliyun.com +zillow.com +ptp24.com +ybpangea.com +go2speed.org +hgads.com +gameloft.com +wt-data.com +tbccint.com +deadspin.com +googlehosted.com +protrade.com +gammaplatform.com +tradedoubler.com +ebay.it +gfycat.com +goadservices.com +radikal.com.tr +crashplan.com +googlezip.net +embedly.com +tqn.com +m6d.com +thechive.com +rantsports.com +bluestacks.com +kiosked.com +dailyfinance.com +cafepress.com +digitru.st +s-nbcnews.com +redrock-interactive.com +chicagotribune.com +turnerstoreonline.com +boston.com +kotaku.com +cnnnewsource.com +real.com +clickability.com +netdna-ssl.com +comodo.com +google.dk +ehow.com +updaterex.com +mozillamessaging.com +and.co.uk +fastcompany.com +genk.vn +github.io +vineapp.com +securedvisit.com +feedly.com +astpdt.com +allstate.com +wal.co +hurpass.com +squarespace.com +politico.com +peel-prod.com +cleanprint.net +groupme.com +techtudo.com.br +sessionm.com +vzwwo.com +mentad.com +jezebel.com +mercent.com +rovio.com +wixstatic.com +bingj.com +targetix.net +amzn.to +espn.com.br +ign.com +adfox.ru +kelkoo.com +reference.com +runadtag.com +myswitchads.com +fqrouter.com +saymedia.com +xhcdn.com +nymag.com +nba.com +polarmobile.com +snapengage.com +swoop.com +vbulletin.com +leafletjs.com +mlstatic.com +s-microsoft.com +terra.com.br +drudgereport.com +sabah.com.tr +sporx.com +boomtrain.com +ad-maven.com +bloglovin.com +swypeconnect.com +vui.vn +mynet.com +splash-screen.net +more-results.net +tunein.com +google.com.my +proptp.net +uol.com +oppuz.com +castaclip.net +errorception.com +lexity.com +dreamsadnetwork.com +duckduckgo.com +naver.com +adobesc.com +pandasoftware.com +kiloo.com +sunbeltsoftware.com +logentries.com +rtbsrv.com +quantcast.com +providesupport.com +vox.com +emodio.com +advconversion.com +qpic.cn +wellsfargo.com +browser-update.org +zenguard.biz +boostadvtracking.com +samsungcloudsolution.com +shoprunner.com +tinnong247.net +intermarkets.net +worthly.com +mol.im +likes.com +fmpub.net +maxthon.com +edigitalsurvey.com +servingrealads83.com +163.com +jump-time.net +pornmd.com +goal.com +mynet.com.tr +ancestry.com +dermstore.com +easybreathe.com +box.net +mycdn.me +etahub.com +payclick.it +blip.tv +adrsp.net +apigee.net +extensionanalytics.com +sayyac.net +upsight-api.com +centauro.com.br +ebay.co.uk +espn3.com +wix.com +msfsob.com +aboutads.info +eproof.com +editmysite.com +trrsf.com +meltdsp.com +zaloapp.com +secondspace.com +keezmovies.com +movieseum.com +lockerdome.com +jsrdn.com +ad6media.fr +alephd.com +spankwire.com +virgilio.it +everyplay.com +tbcdn.cn +targetimg2.com +horsered.com +ally.com +siftscience.com +hotelurbano.com +dellsupportcenter.com +abcnews.com +adsmarket.com +repubblica.it +netflix.net +medleyads.com +richmetrics.com +phonepower.com +picadmedia.com +imgsmail.ru +sonicwall.com +theblaze.com +targetimg3.com +msocdn.com +luyou360.cn +gittigidiyor.net +mlapps.com +dynectmedia6degrees.com +resultspage.com +goodgamestudios.com +reamp.com.br +foxsports.com +burt.io +feiwei.tv +shareth.ru +espnfrontrow.com +ermisvc.com +w3i.com +publichd.eu +exct.net +24hstatic.com +buscape.com +foxnewsinsider.com +viewmixed.com +redtubefiles.com +webssearches.com +yelpcdn.com +adultfriendfinder.com +lavanetwork.net +fb.com +france24.com +rockyou.com +jwplatform.com +customersvc.com +targetimg1.com +extremetube.com +pandasecurity.com +indiatimes.com +venturecapitalnews.us +brand.net +4shared.com +cnt.my +pictela.net +mulctsamsaracorbel.com +ymail.com +learni.st +youronlinechoices.com +tinypic.com +mega.co.nz +bostonglobe.com +naturalon.com +atil.info +lavamobiles.com +hizliresim.com +friendfeed.com +fame10.com +sheknows.com +cootek.com +usekahuna.com +zelfy.com +friv.com +expedia.com +egistec.com +espnscrum.com +jsadapi.com +worldcat.org +clovenetwork.com +mandrillapp.com +microad.jp +allrecipes.com +tuoitre.vn +qhimg.com +catsupagedwelcome.com +realclearpolitics.com +weheartit.com +pub2srv.com +trackerfix.com +apps.fm +rnengage.com +myfitnesspal.com +begun.ru +videologygroup.com +weather.gov +dmtracker.com +ew.com +foxnewsgo.com +emailsrvr.com +washingtontimes.com +bleacherreport.net +box.com +xtify.com +ppjol.com +sweet-page.com +nt.vc +adshostnet.com +alpha00001.com +startappexchange.com +shareasale.com +sexypartners.net +superuser.com +windowssearch.com +torrentsmd.com +astromenda.com +phpbb.com +openxadexchange.com +hubrus.com +threattrack.com +ravenjs.com +shbdn.com +ghostery.com +rottentomatoes.com +uc.cn +comcast.com +voxmedia.com +sony.com +gaytube.com +chaordicsystems.com +answerscloud.com +tru.am +truste-svc.net +xtube.com +jalopnik.com +torchbrowser.com +lifefactopia.com +huluim.com +clicksor.com +awin1.com +captifymedia.com +realmediadigital.com +ttnet.com.tr +ebay.de +coullmedia.com +stopbullying.gov +foodnetwork.com +dana123.com +guardian.co.uk +1688.com +adrttt.com +skinected.com +myimagetracking.com +mercadoclics.com +mmcdn.cn +wmflabs.org +gorillanation.com +ppjol.net +thescore.com +authorize.net +milliyetvideo.com +peeperz.com +apptap.com +wikia-beacon.com +innity.com +yourjavascript.com +ad120m.com +milliyetemlak.com +blizzard.com +cnnmexico.com +acer.com +peel.com +h3q.com +popcash.net +nest.com +bitdefender.com +newsvine.com +yify-torrents.com +porniq.com +umbel.com +wikia.com +viafoura.com +skim.gs +quickbooks.com +likes-media.com +nflcdn.com +baza.vn +sojern.com +cimcontent.net +minireklam.com +prq.to +lifescript.com +reklamz.com +buysub.com +pbwstatic.com +etbxml.com +outfit7.com +rt.com +ig.com.br +servedbyopenx.com +adtechjp.com +cashtrafic.info +gnu.org +mobilecore.com +thepiratebay.se +magnetic.is +estat.com +oasgames.com +viafoura.net +wp.me +lpsnmedia.net +ssuggest.com +plex.tv +gosquared.com +r7.com +yellowpages.com +exacttarget.com +9cache.com +suproo.com +springboardplatform.com +realsimple.com +gazetevatan.com +wikiquote.org +brtstats.com +eggnogthrushdeemster.com +samsungdm.com +allshareplay.com +serverfault.com +usatoday.net +assetfiles.com +youversionapi.com +espn.com.au +exip.org +youporngay.com +clixmetrix.com +kixer.com +nict.jp +cnn.it +wikihow.com +ebdr2.com +linkhay.com +rollingstone.com +usa.gov +dowjoneson.com +alljoyn.org +parentalcontrolbar.org +mediasoul.net +livestrong.com +instacontent.net +securestudies.com +theglobeandmail.com +microsoftonline-p.com +tapstream.com +wsj.net +kickstarter.com +ntius.com +1iota.com +teamskeetimages.com +yimgr.com +userapi.com +datasphere.com +donation-tools.org +compare-electronics.net +aliyuncs.com +experian.com +optimost.com +audienceamplify.com +realtor.com +soha.vn +alipay.com +shape.com +under-myscreen.be +blogcdn.com +socialreader.com +flipkart.com +ticketmaster.com +photobucket.com +nationalreview.com +pusher.com +hobwelt.com +ptp123.com +validwin.com +guim.co.uk +adersite.com +popsugar.com +icptrack.com +stackauth.com +laban.vn +cbc.ca +wellsfargomedia.com +tccdn.com +mathoverflow.net +honcode.ch +msnbc.com +delivery.net +oclasrv.com +ibm.com +merriam-webster.com +firefox.com +trib.al +remat.ca +hao123.com +qadserve.com +tightendjump.com +accesshollywood.com +idqqimg.com +styleblazer.com +sessioncam.com +sendgrid.net +newnext.me +xiami.com +force.com +clkads.com +reacheyes.net +ngaynay.vn +alimama.com +miniclip.com +adbutter.net +ipecho.net +mediawhite.com +istockphoto.com +intercom.io +microsoftstore.com +bdstatic.com +virgul.com +eloqua.com +sling.com +glispa.com +vice.com +conduit-services.com +embed.ly +nflxvideo.net +ambientdigitalgroup.com +travelzoo.com +sfdict.com +footlocker.com +zgncdn.com +bongacams.com +igodigital.com +footballfanatics.com +feedjit.com +adfrontiers.com +sonos.com +thefreedictionary.com +fitbit.com +health1st.com +switchadhub.com +ozy.com +9gag.tv +prnewswire.com +ntv.io +arkadiumhosted.com +cdc.gov +matrixspa.it +sfgate.com +boswp.com +buzzdock.com +mediaoptout.com +uploaded.net +openh264.org +bossip.com +valuepubmedia.com +crowdignite.com +jivox.com +ntvspor.net +haber7.com +winaffiliates.com +extreme-dm.com +awempire.com +scene7.com +ustiming.org +travelandleisure.com +track8172.com +playwire.com +cbslocal.com +thegioicongai.net +genericlink.com +veinteractive.com +wayfair.com +zap.com.br +webcollage.net +oclaserver.com +askubuntu.com +netshoes.com.br +microad-cn.com +heapanalytics.com +cxt.ms +ebit.com.br +economist.com +csctrustedsecure.com +abalo.vn +weeklystandard.com +gamek.vn +highcpms.com +esm1.net +dolphin-browser.com +myfoxny.com +castplatform.com +snapdoapp.com +zillowstatic.com +smh.com.au +craigconnects.org +tealiumiq.com +dlink.com +gifts.com +westelm.com +reporo.net +products-marketplace.com +howlifeworks.com +livejasmin.com +nflxext.com +usps.com +torrentz.eu +siteblindado.com +haydaygame.com +linkz.net +ad-center.com +iana.org +www.net.cn +roimediadigital.com +ebay.ca +wikisource.org +shopathome.com +giphy.com +vagas.com.br +radiotime.com +lolstatic.com +smithmicro.com +mangomediaads.com +general-marketplace.com +tellaparts.com +afilio.com.br +tindersparks.com +hurriyetaile.com +redlaser.com +baomoi.com +kataweb.it +dreamstime.com +metacritic.com +owneriq.com +scribblelive.com +abuse-lawyer.com +openstat.net +staticsfly.com +vzw.com +dmtio.net +wildgames.com +bizo.com +verizonwireless.com +goember.com +reduxmediagroup.com +examiner.com +txmblr.com +nasdaq.com +serve-sys.com +mtvnservices.com +ebay.in +adocean.pl +ebay-us.com +amap.com +auditude.com +frameddisplay.com +weebly.com +webscorebox.com +update-apps.com +bazoocam.org +ammadv.it +rivals.com +omniata.com +trrsf.com.br +sociaplus.com +mediav.com +adxpose.com +libero.it +bigfineads.com +realitytraffic.com +fqtag.com +cobaltgroup.com +neverblue.com +atlassolutions.com +thongtinnonghoi.com +skyfire.com +wpcomwidgets.com +q1media.com +filmifullizle.com +lglime.com +bigpara.com +spiceworks.com +crunchbase.com +wxug.com +zap2it.com +bdimg.com +onclasrv.com +adultadworld.com +jd.com +freegeoip.net +360buyimg.com +webtrekk.net +eastbay.com +mochiads.com +standard.co.uk +data-slimspots.com +xinhuanet.com +telemetrytaxonomy.net +sumome.com +flixcart.com +whatsapp-sharing.com +linkury.com +epom.com +imp-serving.com +swiftypecdn.com +jntwrk.com +geoplugin.net +mpstat.us +adobelogin.com +yunos.com +bfi0.com +laiwang.com +lduhtrp.net +muachung.vn +slickdeals.net +r7ls.net +kakao.com +cookinglight.com +ahalogy.com +bldrdoc.gov +adacado.com +mobytrks.com +caliser.com +hubspot.net +vastglows.com +msn.com.br +adweek.com +lostwaldo.com +xunlei.com +icmwebserv.com +ebaymotorsblog.com +mapbox.com +em.io +nintendowifi.net +qhmsg.com +thoughtleadr.com +ebay.fr +rmlacdn.net +refinery29.com +mtv.com +grooveshark.com +cambio.com +viaf.org +samsungcloudsolution.net +mlb.com +ifunny.mobi +j2inter.com +tinchieu.com +blinkx.com +bantintuoitre.com +kqzyfj.com +displaymarketplace.com +ebay.com.au +mobisla.com +crateandbarrel.com +livejournal.com +trove.com +x1cdn.com +iminent.com +fastcodesign.com +appboy.com +savefront.com +quickheal.com +spilgames.com +rounds.com +mediander.com +last.fm +goforandroid.com +panthercdn.com +stylelist.com +ojrq.net +bluelionsports.com +awltovhc.com +mologiq.net +bdnsrt.org +amxdt.com +swiftype.com +installshield.com +jscods.cf +reviewed.com +onswipe.com +loggly.com +timeanddate.com +hyprmx.com +elasticbeanstalk.com +fotomac.com.tr +southernliving.com +google.com.ph +vdopia.com +breitbart.com +adextent.com +akafms.net +inner-active.mobi +alisoft.com +sohu.com +sayyac.com +gsimedia.net +whstatic.com +zanox.ws +filepicker.io +youtube-mp3.org +rtbserver.com +bitgravity.com +trafficholder.com +wattpad.com +audioscrobbler.com +networkadvertising.org +admaster.com.cn +xaxis.com +icontact.com +fanatik.com.tr +9apps.com +rd.com +siecdn.com +hootsuite.com +yenibiris.com +sinajs.cn +gettyimages.com +anrdoezrs.net +suggest.com +craveonline.com +boxofficemojo.com +recode.net +postimg.org +mlstat.com +creafi-online-media.com +bstatic.com +localresponse.com +mahmure.com +mybrowserbar.com +fastcocreate.com +weibo.cn +battle.net +gscontxt.net +trvl-media.com +madamenoire.com +onelouder.com +4at5.net +deadline.com +thefind.com +trbimg.com +ironlionfun.com +craigslistjoe.com +bkav.com.vn +livelook.com +badoo.com +abc.net.au +dailycaller.com +skeettools.com +homedepot.com +netshelter.net +plista.com +internetat.tv +xahoi247.net +hearstmags.com +quettra.com +arstechnica.com +womanitely.com +ebay.be +startv.com.tr +pokki.com +ariamax.it +mesh.com +oovoo.com +delta-homes.com +cnnfn.com +huffingtonpost.ca +incmd10.com +mkk.com.tr +instantservice.com +esquire.com +edgefonts.net +zopim.io +huffingtonpost.co.uk +foxydeal.com +google.pl +seatgeek.com +nytstore.com +rapidgator.net +garmin.com +adschoom.com +sitestat.com +ebay.at +gamespot.com +cookingsubstitute.com +mncdn.com +placelocal.com +feedsportal.com +medium.com +gltrkk.net +health.com +247wallst.com +mackolik.com +stylemepretty.com +savemyshows.com +radiobeat.com.br +tqlkg.com +lookout.com +invisionpower.com +onedio.com +milliyet.tv +mirror.co.uk +httptrack.com +travel-assets.com +productsmagazines.com +theweek.com +paragonads.vn +audiencemanager.de +nytm.org +ftjcfx.com +product-subsitute.net +spiegel.de +pushwoosh.com +denverpost.com +autoblog.com +whitehouse.gov +disney.com +ning.com +funshion.net +google.se +cozi.com +pbs.org +oaspapps.com +adrcdn.com +dishaccess.tv +houzz.com +behance.net +mercadolivre.com.br +escinteractive.com +cedexis.org +oui-0x00199d.com +dewmobile.net +food-substitute.net +gsspat.jp +vizejs.info +wearemadeinny.com +systweak.com +ministerial5.com +adbroker.de +intentmedia.net +websosanh.vn +delvenetworks.com +barnesandnoble.com +fontdeck.com +barrons.com +thetrafficstat.net +paypal-communication.com +visilabs.com +pastebin.com +ic-live.com +yakala.co +instyle.com +ebay.com.hk +vietnamnet.vn +match.com +zenmate.com +href.asia +icpsc.com +csmonitor.com +dictionary.com +kii.com +districtm.ca +verizon.net +billmelater.com +nbc.com +foodandwine.com +gssprt.jp +pushbullet.com +dianomi.com +kargo.com +xhamsterpremiumpass.com +gtags.net +fazenda.gov.br +southwest.com +wikibooks.org +netd.com +canonical.com +jeep.com +go2rewards.com +incommon.org +mcclatchydc.com +orkut.com.br +isidewith.com +nwps.ws +pcworld.com +zlcdn.com +dishanywhere.com +nhaccuatui.com +trw12.com +rcsmetrics.it +atv.com.tr +octoshape.net +3.cn +burakoyunda.net +kimia.es +adage.com +meraki.com +clickbank.net +adsdumpo.com +bstk.co +jscripts.org +niziot.com +ebay.com.cn +bhaskar.com +ebay.ie +ivcbrasil.org.br +pixfuture.net +mercadolibre.com +chaturbate.com +thesyndicationserver.co.uk +cbs.com +ipredictive.com +bongdaplus.vn +tsn.ca +xiaomi.net +7eer.net +verizon.com +yieldify.com +niwali.com +ibtimes.com +wmt.co +sinaimg.cn +rnmd.net +hastrk2.com +ad-sys.com +epoch.com +v9.com +themeforest.net +bongda.com.vn +trb.com +sportscenter.com +flowplayer.org +inquisitr.com +smartling.com +sancdn.net +grab-media.com +vinacaptcha.com +offthebus.org +telemetryaudit.com +admaxserver.com +teamespn.com +51.la +controlyourtv.org +myrecipes.com +cqq5id8n.com +boomads.com +kiloo-games.com +ophan.co.uk +adspirit.net +certona.net +ptp33.com +placehold.it +troveread.com +ixxx.com +genieesspv.jp +genieessp.jp +jscache.com +tracking8171.com +mom.me +pgcdn.com +quikr.com +aexp-static.com +fortiguard.net +shutterstock.com +thebighits.com +vccorp.vn +thesaurus.com +leadpages.net +myhomeideas.com +vserv.mobi +planalto.gov.br +blogads.com +incmd03.com +indianexpress.com +hurriyetdailynews.com +boombeachgame.com +propelplus.com +golf.com +fanatics.com +getm.pt +sonyentertainmentnetwork.com +ebaypartnernetwork.com +pricedetect.com +accountonline.com +commissionlounge.com +rutarget.ru +trendcounter.com +umunu.com +lasvegassun.com +voipwelcome.com +aztecbe.com +google.com.tw +shopperconnect.com +ekolay.net +pandonetworks.com +genieessp-a.com +salesforce.com +explabs.net +b-io.co +epsihost.com +variety.com +stripe.com +tripadvisor.com.tr +vrvm.com +ilibr.org +dtravelconnection.com +grnh.se +olx-st.com +rr.com +guardianapps.co.uk +emgn.com +medscape.com +mashery.com +sh.st +wondershare.com +eventbrite.com +invodo.com +netbookmedia.com +falecomog1.com.br +criminalcasegame.com +peopleenespanol.com +potterybarn.com +tndmnshb.com +zoopla.co.uk +etracker.de +xiti.com +makers.com +35go.cn +sa-live.com +jpush.cn +goodwaygroup.com +trustkeeper.net +unicornmedia.com +xidx.org +suntimes.com +nesine.com +hurriyettv.com +thanhnien.com.vn +imshopping.com +mac.com +intag.co +affinity.com +tvinteractive.tv +corriere.it +lowermybills.com +truthrevolt.org +gooncheck.com +glassdoor.com +mgtracker.org +motherjones.com +osdimg.com +govdelivery.com +statig.com.br +funweek.it +sina.com.cn +mathjax.org +blogherads.com +agame.com +curse.com +glowingskinsecret.com +betweendigital.com +cxpublic.com +enbac.com +gmodules.com +easytomessage.com +smithsonianmag.com +trackimpression.com +best-products-review.com +jasmin.com +c-launcher.com +dominos.co.uk +sahadan.com +go2cloud.org +peakgames.net +ad127m.com +bridgetrack.com +miniclipcdn.com +adrcntr.com +arcadeweb.com +springserve.com +tatami-solutions.com +iconosquare.com +zeobit.com +infowars.com +agilone.com +pravda.ru +solo-launcher.com +lporirxe.com +google.com.co +myway.com +getfirefox.com +messagelabs.com +mabaya.com +onesmartpenny.com +investors.com +logitech.com +turbobit.net +darchermedia.com +hepsiburada.net +minecraftevi.com +beenverified.com +getresponse.com +perfectnavigator.com +mortgagesmade.com +loans-made.com +monstersandcritics.com +shifen.com +corriereobjects.it +consumerviews.net +businesswire.com +capitalone.com +oyunasi.com +contentspread.net +enoratraffic.com +pinger.com +mobogarden.com +vyped.com +danviet.vn +vresp.com +dellbackupandrecovery.com +britannica.com +hepsiburada.com +alkislarlayasiyorum.com +wordego.com +likesharetweet.com +gmarket.co.kr +kastatic.com +idea-marketplace.com +digitaltrends.com +sony.tv +buzzwok.com +streamtheworld.com +jdoqocy.com +boingo.com +aaplimg.com +camads.net +vitv.it +adrtx.net +default-search.net +ulive.com +updaterss.com +173uu.com +maxworkouts.com +srpx.net +ignimgs.com +cetrk.com +nhl.com +stats.com +evergage.com +bostonherald.com +ebay.es +townnews.com +runhaven.com +rongbay.com +airfrance.com +stathat.com +haaretz.com +intercomcdn.com +baronsoffers.com +gr-assets.com +ebayclassifieds.com +getadblock.com +gazzetta.it +hshh.org +blogsmithmedia.com +pubgears.com +medicinenet.com +giadinh.net.vn +naytev.com +aol.it +brasil.gov.br +beeg.com +google.co.ve +kitchendaily.com +farolatino.com +allyou.com +efe.com +dogpile.com +goroost.com +seccint.com +xmarks.com +ibsys.com +offer-dynamics.com +reponets.com +yes.my +bbystatic.com +dc-storm.com +zapps.vn +staples.com +kyodonews.jp +pcmag.com +santandernet.com.br +jdownloader.org +google.co.id +whitepages.com +newshunt.com +discover.com +intensedebate.com +onlineregister.com +scubl.com +webovernet.com +telize.com +vanityfair.com +rcs.it +demonoid.com +iol.it +dallascowboys.com +bhg.com +innovatenetworks.com +fout.jp +rackspacecloud.com +irna.ir +pornhubpremium.com +cdn-apple.com +flxpxl.com +searchignite.com +caixa.gov.br +barclaycardus.com +nytco.com +htcsense.com +everydayhealth.com +etracker.com +heraldonline.com +grainger.com +ptpcpm.com +comscore.com +t4ft.de +bluehost.com +state.gov +cometourgeorgia.com +affinitymatrix.com +trulia.com +uniblue.com +trbas.com +synology.com +llbean.com +spinr.in +miaozhen.com +hurriyetkampus.com +games.com +jsonip.com +247inc.net +adshexa.com +sfx.ms +dbjhr.com +putags.com +pickmeup-ltd.com +trafficshop.com +line-apps.com +ampxchange.com +serpro.gov.br +preyproject.com +imrk.net +att.net +healthination.com +mysearchdial.com +adledge.com +baixaki.com.br +snxd.com +ionicframework.com +ebay.com.my +scribdassets.com +lp4.io +polyvore.com +rvty.net +crawlability.com +freebase.com +radio.com +videohub2.tv +adziff.com +brazzers.com +sndimg.com +kiplinger.com +vagalume.com.br +advance.net +coastalliving.com +tfd.com +apnewsregistry.com +mediadecision.com +leadboltapps.net +videolan.org +trustlogo.com +exactag.com +twitchmediagroup.com +theweathernetwork.com +webrootcloudav.com +auctiva.com +bangmychick.com +zeusclicks.com +irs.gov +urbandictionary.com +minus.com +4wmarketplace.com +departures.com +stylebistro.com +sweetim.com +junbi-tracker.com +brandsmind.com +heatmap.it +amplitude.com +mdotm.com +marca.com +tkqlhce.com +twonky.com +sendo.vn +usnews.com +cyberpatrol.com +hgtv.com +kazhifu.com +eva.vn +thestreet.com +wsimg.com +bidvertiser.com +cpmshield.com +sunset.com +lonny.com +df-stream.net +liveclicker.net +upsellit.com +vocalocity.com +adxcore.com +nature.com +ustream.tv +advertise.com +eblastengine.com +qiyi.com +thebookinsider.com +supert.ag +dota2.com +thepaperboy.com +breakingburner.com +ilovevideo.tv +tvguide.com +shebudgets.com +jstor.org +toplist.cz +zoom.com.br +wpdigital.net +baltimoresun.com +listrakbi.com +meetic-partners.com +piriform.com +jobsite.co.uk +metro.co.uk +dumedia.ru +shld.net +tribdss.com +essence.com +rutor.org +hotels.com +chinacache.com +change.org +britishairways.com +vexigo.com +ilivid.com +espnshop.com +newsweek.com +myspacecdn.com +cbox.ws +getpocket.com +primelocation.com +asacp.org +theatlanticwire.com +coreclickhoo.com +mythingsmedia.net +indianrail.gov.in +tvbythenumbers.com +dilcdn.com +babcdn.com +kejet.net +gameforge.com +urbanoutfitters.com +dpmsrv.com +where.com +sciencedirect.com +netnanny.com +appfireworks.com +beygir.com +evidon.com +fplive.net +tbo.com +loc.gov +demonoid.me +prchecker.info +linksalpha.com +zazzle.com +torrentbay.to +mobilityware.com +proximic.com +kanald.com.tr +mandatory.com +charter.net +cracked.com +secure-trkr.com +comm100.com +wordreference.com +macropinch.com +pages03.net +newgenstatsnet.com +apikik.com +iolo.com +ashleymadison.com +trello.com +abril.com.br +neon-lab.com +extend.tv +hurriyetemlak.com +mythings.com +vmn.net +ipromote.com +divx.com +philly.com +so.com +yelp-ir.com +imgiz.com +simplemachines.org +evcdn.com +lowes.com +ssacdn.com +vidobu.com +cmail2.com +snidigital.com +hurriyetcocukkulubu.com +entrepreneur.com +infostrada.it +opinionlab.com +pubventuresmedia.com +anm.co.uk +industrybrains.com +tahminkolik.com +vulture.com +wanelo.com +hdonline.vn +garena.vn +interfax.com +j.mp +houstontexans.com +thisismoney.co.uk +citrix.com +puu.sh +crobo.com +bamstatic.com +steelers.com +edgecastdns.net +splashtop.com +cbsistatic.com +api.tv +reachadv.it +extole.com +nasiltv.com +dnsomatic.com +online.gov.vn +plimplim.com.br +broadcastingcable.com +picsart.com +thisoldhouse.com +bigcommerce.com +philadelphiaeagles.com +adoburcrv.com +ranker.com +nsimg.net +smartcampaign.it +greatschools.org +iheartradio.com +patriots.com +qvc.com +9gaging.com +ziplist.com +rocketadserver.com +revolutiongolf.com +scrippscontroller.com +starbucks.com +delta.com +soft365.com +staticpm.com +contadd.com +cpleft.com +pricejs.info +newsmaxhealth.com +moborobo.com +fullhdfilmizle.com +pulse.io +usgs.gov +mediaite.com +tampabay.com +tribune.com +bandito.org +allmusic.com +ipaddresslabs.com +popcap.com +nasil.tv +samsungmobile.com +shns.com +ttinline.com +mailtravel.co.uk +autocompleteplus.com +sawpf.com +sozcu.com.tr +tomshardware.com +yp.com +askmen.com +eventful.com +ambientplatform.vn +bengals.com +bizjournals.com +blogcu.com +dpbolvw.net +undertonevideo.com +newseum.org +friendschecker.com +ultradns.co.uk +cnnarabic.com +adultwebmasternet.com +nai.com +createjs.com +adkontekst.pl +sears.com +cdnetworks.net +cloudsponge.com +intuitstatic.com +hissage.net +databrain.com +lifespan.com +sportstadio.it +xfreeservice.com +buffalobills.com +dsiteproducts.com +exad.me +pqarchiver.com +hellobar.com +chron.com +quifinanza.it +downloadhelper.net +surveey.com +marriott.com +wunderlist.com +elpais.com +xe.com +c-span.org +omnitagjs.com +zonealarm.com +okmagazine.com +opentable.com +dsg.com +spamexperts.com +pbteen.com +nanglobal.com +zip.net +oldtiger.net +extratorrent.cc +wikinews.org +teach.org +basecamp.com +medianetadvertising.com +colts.com +healthgrades.com +williamhill.it +newegg.com +cpmterra.com +ads-creativesyndicator.com +cars.com +quickplay.com +ireport.com +wandoujia.com +wikiversity.org +privateinternetaccess.com +clevelandbrowns.com +upi.com +uptolike.com +nflshop.com +imgur-ysports.com +earthlink.net +dtvbb.tv +taringa.net +ziffdavis.com +eddiebauer.com +surveygizmo.com +ehowcdn.com +food.com +heyzap.com +popdust.com +dtinews.vn +ramp.com +foxitcloud.com +ampdesk.com +srvabc.com +pawnation.com +mediaset.it +hi-mediaserver.com +miamidolphins.com +blogher.com +babbel.com +fptad.net +pagesix.com +sendmessagebox.com +inatjs.info +patch.com +mgccw.com +tmgrup.com.tr +paradox.com +qihoo.com +hurriyetoto.com +deca.vn +deviantart.net +climatempo.com.br +adhexa.com +foreignpolicy.com +tp-link.com +miamiherald.com +madadsmedia.com +fyleio.com +llnw.net +ebay.nl +nbcconnecticut.com +opensharing.org +d-nb.info +flyertown.ca +selectmedia.asia +quicktransmit.com +bongdaso.com +rmgserving.com +profootballhof.com +denverbroncos.com +mydomainadvisor.com +meetup.com +tracker-ccc.de +united.com +meredith.com +copyright.com +bahis-sirketleri.com +yardbarker.com +untd.com +jaguars.com +npario-inc.net +agoda.com +miibeian.gov.cn +thegatewaypundit.com +europa.eu +kingsoftstore.com +usbank.com +fxdepo.com +google.be +msdn.com +oroll.com +payn.me +l1o0l11lo11011o.com +nationalgeographic.com +scrippsnetworks.com +sbitinjs.info +ibxads.com +servebom.com +yesware.com +ibtracking.com +bradesco.com.br +newyorkjets.com +vidyoda.com +liebao.cn +vidmate.net +getsatisfaction.com +packers.com +slacker.com +dtzads.com +appwork.org +mailonsunday.co.uk +ixigo.com +wurfl.io +limelight.com +tbcache.com +applifier.info +suckhoedoisong.vn +yelp-press.com +siviaggia.it +ibxk.com.br +toofab.com +e7r.com.br +medio.com +meccahoo.com +ticketexchangebyticketmaster.com +livingsocial.com +kcchiefs.com +aliyuncdn.com +musixmatch.com +chargers.com +3q.com.vn +acunn.com +cmail1.com +gez.io +madmimi.com +nflrush.com +aimatch.com +okcupid.com +thenextweb.com +buonissimo.org +onsugar.com +savefrom.net +adopshost1.com +perfectlytimedpics.com +zini.vn +yelp-support.com +ibpxl.com +italiaonline.it +tonefuse.com +veeseo.com +affiz.net +newsmax.com +liqwid.net +prezi.com +emediate.dk +customer.io +dilei.it +martiniadnetwork.com +drtuber.com +campaignism.com +ccbill.com +chicagobears.com +frontdoor.com +adrdgt.com +kokteyl.com +adversal.com +admailtiser.com +upsjobs.com +ikea.com +teamfanshop.com +v3cdn.net +500px.com +gfsrv.net +lightningnewtab.com +anycastcdn.net +uzmantv.com +evite.com +ati-host.net +globaltestmarket.com +detroitlions.com +upworthy.com +msgamestudios.com +ethn.io +e2ma.net +marketgid.com +maudau.com +thedenverchannel.com +softonic.com +crowdynews.com +mediamond.it +dsusw.net +advertserve.com +gmx.com +as.com +stackapps.com +akilli.tv +simplytechnology.net +fastcoexist.com +celebuzz.com +skor.tv +cheetahmail.com +ibtimes.co.uk +lovedgames.com +vevo.com +bd-pl.com +push.io +villarenters.com +uadx.com +emarbox.com +musica.com.br +kmplayer.com +parperfeito.com.br +actnx.com +clientstaticserv.com +internethaber.com +codecanyon.net +addthiscdn.com +bbc.net.uk +statigr.am +centurylink.net +kavanga.ru +aviary.com +1and1.com +hrblock.net +titansonline.com +sellpoints.com +sharesdk.cn +zdnet.com +targetspot.com +hrblock.com +adk2.net +traffichunt.com +vuigame.vn +ilius.net +vaccint.com +adsnative.com +githubapp.com +tvyo.com +privatehomeclips.com +netmahal.com +shazam.com +arcadesafari.com +panthers.com +peoplestylewatch.com +ebaycareers.com +interoperabilitybridges.com +izlesene.com +adsquangcao.com +bloombergview.com +sportingnews.com +dalealplay.com +chinhphu.vn +pixlr.com +ehownowcdn.com +nflyouthpd.com +wenn.com +mackolikcomplex.com +adgorithms.com +mojang.com +activebeat.com +verticalresponse.com +sftcdn.net +bancobrasil.com.br +bitcoin.org +star-telegram.com +sigfig.com +himediads.com +redd.it +anametrix.com +mimecast.com +baidu.com.eg +endeavor.org.tr +thebrittanyfund.org +crdui.com +adspeed.net +tpb.vn +redskins.com +xrosview.com +mercurynews.com +atlantafalcons.com +xiaomi.com +readspeaker.com +dowjones.com +casa.it +hubapi.com +cox.net +arcgisonline.com +luxup.ru +metanetwork.com +hatid.com +wiley.com +adplugcompany.com +neon-images.com +vikings.com +spilcdn.com +azcardinals.com +nascar.com +nationaljournal.com +webink.com +genesismedia.com +webtype.com +torcache.net +ycombinator.com +wdtinc.com +autodesk.com +abc.com +fortinet.net +sellathon.com +mocean.mobi +apiok.ru +revenuehits.com +launchpad.net +netmining.com +parentsociety.com +ribob01.net +diply.com +operamini.com +shopper-pro.com +userzoom.com +22find.com +condenast.com +tinypass.com +speedshiftmedia.com +localworld.co.uk +vetstreet.com +newjobs.com +icq.com +eorezo.com +neworleanssaints.com +zulily.com +buccaneers.com +pxlad.io +edintorni.net +xenforo.com +liveleak.com +google.com.ua +thethao247.vn +admedia.com +pagelyhosting.com +spingo.com +rxlist.com +weightlosspath.com +noviretrack.com +mp3skull.com +lenzmx.com +theupsstore.com +gamblingtherapy.org +ntent.com +extremetracking.com +cmgdigital.com +onestat.com +custhelp.com +wunderloop.net +e-kolay.net +youdao.com +motorlife.it +proxysandy.com +camplace.com +worldlingo.com +nordeus.com +yandexadexchange.net +cubicleoffers.com +hearstnp.com +a-ads.com +macys.com +adsdk.com +inclk.com +synacor.com +atomex.net +careland.com.cn +linuxmint.com +php.net +rovion.com +informaction.com +justjared.com +biblegateway.com +fansided.com +posterous.com +trustpilot.com +rcsobjects.it +doji.vn +rapsio.com +travelers.com +tebilisim.com +terra.com +youku.com +leanplum.com +lgtvsdp.com +fica.vn +phim3s.net +aa.com +fbnstatic.com +emedicinehealth.com +perezhilton.com +rtbpop.com +seahawks.com +grapeshot.co.uk +ocsp-responder.com +opta.net +rivalgaming.com +avgmobilation.com +bloglines.com +maxthon.cn +topix.net +google.com.pe +otherlevels.com +google.ie +flux.com +dpcdn.com +eztv.it +stltoday.com +wsjlocal.com +aljazeera.com +dt07.net +vatgia.com +vupdate2.com +nuggad.net +bufferapp.com +hdviet.com +sharedcount.com +pages04.net +cinemanow.com +ad121m.com +dudamobile.com +uimserv.net +gaug.es +flattr.com +wetransfer.net +rdio.com +yts.re +toysrus.com +samsungelectronics.com +webcitation.org +jumptime.com +adswizz.com +dianxin.net +ubertags.com +hubpages.com +mobilethreat.net +mcafeeasap.com +linezing.com +brainjet.com +bluenationreview.com +fda.gov +ihrhls.com +nyti.ms +ladsp.com +luckyorange.com +daringfireball.net +ebaymainstreet.com +forbadeplanhad.com +n-able.com +deezer.com +ebay.com.sg +www.gov.uk +eu-ibi.co.uk +tmocce.com +trackjs.com +tigerrunhigh.com +tcgtrkr.com +torrent.to +adcast.io +ip-api.com +trulia-cdn.com +furious7.com +talktv.vn +tcimg.com +lscdn.net +htimg.net +apartments.com +whsites.net +reflexion.net +kaft.com +quikdisplay.com +uplynk.com +forbesimg.com +sonymobile.com +rediff.com +esrb.org +wnyc.org +stlouisrams.com +zqtk.net +gnd.com +youtubeaccelerator.com +kralfm.com.tr +samsungyosemite.com +autotrader.com +espnradio.com +jswrite.com +ebay.ph +pphosted.com +ul.to +foolcdn.com +mnginteractive.com +walgreens.com +adexprt.com +ppstream.com +dtvce.com +shockpedia.com +walmartstores.com +tapcommerce.com +getsentry.com +sitelock.com +theberry.com +torrentfrancais.com +touchcommerce.com +networksolutions.com +wfrcdn.com +hotjar.com +eksisozluk.com +ifttt.com +fishwrapper.com +sli-spark.com +piratebrowser.com +geoportal3d.com.br +ebay.pl +iqiyi.com +sumotracker.com +pof.com +gifsoup.com +rezserver.com +oclc.nl +adready.com +filehippo.com +ebuzzing.com +optproweb.info +destinationtips.com +nhacso.net +sf49ers.com +yahoo.co.jp +minecraftforum.net +mobclix.com +dogannet.tv +eyedemand.com +brandads.net +ria.ru +jawbone.com +toyota.com +petametrics.com +storify.com +srdrvp.com +brassring.com +wavesecure.com +tmcs.net +appads.com +hao123img.com +mercadolivre.com +scansoft.com +posttv.com +edb.gov.sg +gcion.com +bluetie.com +prntscr.com +cursecdn.com +sinemalar.com +tapas.net +rtbhouse.com +haircolorforwomen.com +youm7.com +talkingpointsmemo.com +yieldselect.com +softonic-analytics.net +lenta.ru +sellpoint.net +gofundme.com +7176.com +boredpanda.com +megafilmeshd.net +qhupdate.com +gomlab.com +f-secure.com +garanti.com.tr +hockeyapp.net +passport.net +cootekservice.com +carmax.com +solidstatenetworks.net +bm23.com +ads-grooveshark.com +gs-cdn.net +playboy.com +gplus.to +postads24.com +kongregate.com +jangonetwork.com +jpost.com +movieclip.com +channelvn.net +revmob.com +windstream.net +smarterremarketer.net +bloomberght.com +uolhost.com.br +rkdms.com +hdnux.com +instapaper.com +bet365.com +hepsibahis6.com +istartsurf.com +graphicriver.net +litecoin.org +officedepot.com +fanpop.com +thedailyeight.com +copy.com +alimama.cn +bnf.fr +hahatimes.com +axf8.net +hollywoodlife.com +hhs.gov +presage.io +bonton.com +tencent.com +hayhaytv.vn +allegro.pl +gamefaqs.com +vtc.vn +ui-portal.de +subscene.com +web.de +g2a.com +cloudcell.com +blog-hits.com +shopping-site-directory.com +casagarage.com +rotoworld.com +cibodistrada.it +ungdungviet.com +gnt.com.br +toptenreviews.com +ntvmsnbc.com +dw.de +slutroulette.com +hao123.com.eg +brsrvr.com +fidelity.com +sky.com +wetransfer.com +siriusxm.com +gamesir.com +a2g-secure.com +ebay.ch +rejuvenation.com +s-msft.com +montiera.com +petrotimes.vn +incredimail.com +adsame.com +otwsftv0.com +shopping-guide-centre.com +mystartsearch.com +tamindir.com +gadgets-buy.net +arcgis.com +incmd05.com +capitalradio.com.tr +blurdev.com +himediadx.com +gamestop.com +maxcdn.com +icbdr.com +nct.vn +militarycity.com +keepvid.com +quickmeme.com +geoadnxs.com +tripadvisor.co.uk +streamprovider.net +livescore.com +tutsplus.com +channelnewsasia.com +thedodo.com +bevomedia.com +ballotpedia.org +sandai.net +tabtimes.com +acint.net +amazonsilk.com +getbootstrap.com +razerzone.com +thestar.com +cdndn.net +expedia.ca +useclearthink.com +grabnetworks.com +bootstlab.com +cnbce.tv +baohay.vn +androidcentral.com +infinitummovil.net +bee7.com +bhphotovideo.com +ad123m.com +stackadapt.com +drp.su +egrana.com.br +collegehumor.com +doviz.com +smi2.net +livescience.com +gq.com.tr +ads-srv.net +smartclick.net +discovercard.com +thomsonreuters.com +ameblo.jp +oyunskor.com +tnetnoc.com +instructables.com +newser.com +book-showroom.com +inrim.it +mfcreative.com +adorika.com +arcadecandy.com +elitedaily.com +blinklist.com +baiducontent.com +ians.in +sophos.com +yuq.me +advanseads.com +oley.com +ndtv.com +123mua.vn +philips.com +promobay.org +techhive.com +appia.com +dogusdergi.com +download.com +e2.tv.tr +000dn.com +bzgint.com +flipkart.net +everyone.net +s2d6.com +teamworkonline.com +onlinewebstat.com +crpnms.com +mayoclinic.org +tradera.com +nasa.gov +healthonnet.org +wwwpromoter.com +inyt.com +emusic.com +pages05.net +media-allrecipes.com +raptr.com +clickprotects.com +cdncontents.com +ygsgroup.com +onescreen.net +bahistuttur.com +pc120.com +htemlak.com +aim.com +fenixm.com +cnbce.com +nzherald.co.nz +zero-team.com +xdeal.vn +winamp.com +emediate.eu +adbooth.com +apollocdn.com +jetpackdigital.com +google.ch +paipaiimg.com +thedatingnetwork.com +copperegg.com +kuwo.cn +rss2search.com +zune.net +hearst.com +attccc.com +rutracker.org +portalsepeti.com +securence.com +memeful.com +bigmir.net +iubenda.com +bustle.com +celebritytoob.com +devour.com +onlinecreditcenter6.com +icq.net +ist-track.com +tmomail.net +champssports.com +w3schools.com +prodigy.net +gadgetspurchase.com +youlamedia.com +connexity.com +buzzcity.net +mm-health.com +uproxx.com +toothbrushguru.com +longurl.it +bongacash.com +turkcell.com.tr +mindjolt.com +6si.com +kraltv.com.tr +sendevent.net +hicloud.com +smowtion.com +subito.it +ted.com +overstock.com +tout.com +badoocdn.com +seedceo.com +fluxstatic.com +mmafighting.com +reutersmedia.net +facebook.com.br +macworld.com +s8.com.br +frontbridge.com +kralpop.com.tr +nationalgeographic.com.tr +inputdatacloud.com +shoppingate.info +thestaticvube.com +tomsguide.com +netteller.com +tecmundo.com.br +pickupcloud.com +express.com +lockhosts.com +slashdot.org +nyaatorrents.info +secureboxes.net +technet.com +clamav.net +avclub.com +viralnewschart.com +fox.com +ipinfodb.com +smilebox.com +uverse.com +renren.com +imageg.net +potterybarnkids.com +inskinmedia.com +cexchange.com +11oyun.com +skorer.tv +bulletinsync.info +realclearmarkets.com +sinemaizle.org +nbcnewyork.com +devicescape.net +factiva.com +shoppingonlinedirectory.com +kralpoptv.com.tr +orcali.com +expedia.co.uk +smarterlifestyles.com +onscroll.com +deximedia.com +sprintpcs.com +nrgbinary.com +usafootball.com +boredlion.com +freemake.com +jango.com +babycenter.com +complex.com +expedia.com.au +htspor.com +itsupport247.net +tiscali.it +todotorrents.com +kijiji.ca +escapehere.com +bb.com.br +census.gov +wallst.com +mortgages-guide.net +clickjogos.com.br +ebay.co.th +goweloveit.info +admagnet.net +podtrac.com +citrixonline.com +video-one.com +p0.com +polarisoffice.com +castradio.net +parkwind.com +rating-widget.com +rockchip.com +citibank.com +splitcamera.com +jam.com.vn +fsdn.com +kayak.com +mastercard.com +lanistaads.com +astbr.com +salesforceliveagent.com +adexprts.com +premiereinteractive.com +yourshoppingoutlet.com +fuq.com +digitaloptout.com +popcrush.com +zst.com.br +mmajunkie.com +google.no +optonline.net +wow.com +css-tricks.com +adnet.vn +snappea.com +siteimprove.com +fbmta.com +meteomedia.com +123phim.vn +latest.com +megatrack.co +9hoho.com +who.int +bzfd.it +research.net +comcastnets.net +boots.com +gocyberlink.com +gotraffic.net +receitas.com +bmwusa.com +bankone.com +mindspring.com +shopping-outlet.net +tubegalore.com +discovery.com +imesh.com +fandango.com +ntvradyo.com.tr +cafef.vn +intencysrv.com +cbsig.net +yieldkit.com +azlyrics.com +ptinews.com +gx101.com +noaa.gov +wajam.com +funshion.com +media1first.com +usadserver.com +olx.pl +pdfcomplete.com +hurlist.com +wipmania.com +intgr.net +whicdn.com +tyroodr.com +joinexpedia.com +myad.vn +shuntv.net +gaytubevideos.com +homepage.com.tr +keywordblocks.com +incapdns.net +google.com.au +ntvsporsmart.com +reelfeed.tv +best-deals-products.com +google.pt +fixyourbloodsugar.com +stylene.net +mozillalabs.com +snapdeal.com +staradvertiser.com +massrelevance.com +playsushi.com +tickld.com +babylon.com +eagnews.org +he.net +lswcdn.net +complexmedianetwork.com +drugs.com +thenation.com +wsjradio.com +webmotors.com.br +expedia.de +websta.me +bomnegocio.com +ya.ru +usyncapp.com +shaw.ca +rcn.com +google.ro +nflcommunications.com +wowway.com +qadabra.com +google.bg +blazing.de +elmundo.es +eba.gov.tr +tfile.me +gameoapp.com +lifestyleasia.com +haivl.com +amplifinder.biz +scarabresearch.com +mydlink.com +morningstar.com +udn.com +dragonbyte-tech.com +livestream.com +yandex.ua +clickfast.co +toledoblade.com +letras.mus.br +wasabii.com.tw +epicgameads.com +hsoub.com +gq.com +joomla.org +dpstack.com +repstatic.it +google.co.jp +offers4u.org +vw.com +srvstatsdata.com +drpsrvr.com +publicidees.com +onavo.com +google.cl +iflscience.com +9to5mac.com +reklm.com +pastebay.net +liftdna.com +google.co.nz +break.com +sprint.com +nikkei.com +viewablemedia.net +sonital.com +ltvcms.com +baltimoreravens.com +inscname.net +abullseyeview.com +doodlemobile.com +3600.com +turnto.com +grindr.com +247-inc.net +ezakus.net +google.at +ntvyayinlari.com +moikrug.ru +sina.cn +readserver.net +ulogix.ru +searchengineland.com +sogoucdn.com +robbreport.com.tr +servicos.gov.br +baynote.net +katespade.com +geniusweekly.com +walmart.com.br +wsjwine.com +360.com +snapdo.com +lienminhhuyenthoai.vn +google.cz +quizzyn.com +inq.com +americanas.com.br +cpnscdn.com +ra47r.com +google.fi +swebdpjs.info +gmx.net +giants.com +washingtonexaminer.com +correios.com.br +mmo-champion.com +rantmovies.com +splkmobile.com +folha.com.br +citrixonlinecdn.com +cam4s.com +drweb.com +ecorebates.com +ostkcdn.com +usa.net +certum.pl +badlefthook.com +wahwahnetworks.com +game-advertising-online.com +anonymox.net +venturebeat.com +tuttur.com +cpaptimes.com +fanduel.com +51y5.net +top100.ru +worldstarhiphop.com +google.sk +trafficforce.com +google.com.sg +admngronline.com +installfarm.com +espncricinfo.com +netshoes.net +trueconf.net +fluentmobile.com +doubleclick.com +maxpointinteractive.com +mchsi.com +hardsextube.com +hthayat.com +dailykos.com +d3js.org +google.co.il +five.tv +intelliad.de +realclearsports.com +songza.com +desmoinesregister.com +omeljs.info +indiegogo.com +hon.ch +newrepublic.com +leadid.com +google.co.th +telmex.com +rvpadvertisingnetwork.com +adreadytractions.com +google.co.hu +carrierzone.com +psafe.com +orange.fr +sharepointonline.com +google.co.kr +townhall.com +sub2tech.com +bemobile.ua +securetve.com +host.sk +bgr.com +xing.com +weddingpaperdivas.com +duba.com +networkmagic.com +hautelook.com +santander.com.br +btbuckets.com +btttag.com +t26.net +pornstargalore.com +coupons.com +usatodayclassifieds.com +meetme.com +bahis-oranlari.com +meteorsolutions.com +demandstudios.com +expedia.co.jp +roomkey.com +vuitruyentranh.vn +madsone.com +adp.com +onlinewebstats.com +spn.com +wynk.in +yandex.com +xfinity.com +hospitalitynet.org +legacy.net +linkbolic.com +donanimhaber.com +xtendmedia.com +newgenonlinesrv.com +srvntrk.com +klm.com +patreon.com +flowstats.net +ehowcommcdn.com +fishbowl.com +mobileiron.com +citi.com +ekomi.de +cooking.com +samplicio.us +quizlet.com +tovarro.com +ultraadserver.com +techradar.com +webpagescripts.net +cpmba.se +yoz.io +meteo.it +webwebget.com +victoriassecret.com +here.com +ebay.vn +craigslist.hk +openxmarket.asia +telus.net +streamcloud.eu +toplist.eu +ilsole24ore.it +ebaumsworld.com +op-cdn.net +newshuntads.com +ticketm.net +google.gr +urbantabloid.com +slideshare.com +ctvnews.ca +bahisfoni.com +google.com.sa +expedia.fr +ayads.co +bayimg.com +irishtimes.com +247sports.com +bayfiles.net +visistat.com +rferl.org +smoothfusion.com +solidoak.com +yhd.com +inboundmx.com +expedia.it +lemonde.fr +ourtime.com +quora.com +mlt01.com +clarin.com +textnow.me +buyvip.com +worldtimeserver.com +newsok.com +motherless.com +octrocdn.com +simply.com +birdstep.com +tudogostoso.com.br +getportal.net +nike.com +localpages.com +schnutzelhuber.com +amkspor.com +bronto.com +pjmedia.com +ezinearticles.com +certsentry.com +tudou.com +lavasoft.com +bahissiteleri.mobi +metrolyrics.com +ifcdn.com +turkiye.gov.tr +epicplay.com +palcomp3.com +hubtraffic.com +vzwfemto.com +moz.com +pornorama.com +amobee.com +sdlcdn.com +ndmdhs.com +mediabong.net +picasasoftware.com +canli-casinositeleri.com +bills.com +recreativ.ru +ideel.com +rockabox.co +rediffmail.com +academia.edu +appsfire.net +bild.com +bnqt.com +phunware.com +reactiongifs.com +suite6ixty6ix.com +meb.gov.tr +psmtp.com +mdpcdn.com +golfchannel.com +goodhousekeeping.com +apply2jobs.com +squidoo.com +usatodayhss.com +clearchannel.com +fbshare.me +advertising-support.com +hp-ww.com +panoramio.com +vube.com +rontar.com +opposingviews.com +nakedtube.com +cartalk.com +atlassbx.com +videotron.ca +mmptrack.com +commercialintegrator.com +stormiq.com +imagesbn.com +arabam.com +wdc.com +profootballfocus.com +smtproutes.org +manta.com +digitaltarget.ru +kansascity.com +nq.com +cameo.tv +poponclick.com +canlirulet-siteleri.com +2sao.vn +ifeng.com +chefscatalog.com +redcross.org +pandora.tv +typepad.com +yemektarifleri.com +bc.vc +eltrafiko.com +gwu.edu +scmp.com +yammer.com +anthropologie.com +gogii.com +uscellular.com +alcatelonetouch.com +kijiji.it +grovupdt.com +maxpreps.com +bttrack.com +socialpointgames.com +studiopress.com +wxbug.com +wpadsvr.com +faithtap.com +echo.msk.ru +casinositeleri.biz +fdnames.com +wikivoyage.org +yazarkafe.com +giga.xxx +detik.com +fc2.com +netcrawl.info +all-free-download.com +gannett.com +glammedia.com +lincoln.com +purewow.com +el-ladies.com +wsjplus.com +speedial.com +ebayadvertising.com +8tracks.com +td.com +instagramfollowbutton.com +securejump.net +rankingames.com +workintelligent.ly +howstuffworks.com +thefashionfanatic.com +staticworld.net +59saniye.com +vzw.net +cisco.com +filmdiziseyret.com +pelmorex.com +congan.com.vn +t24.com.tr +findnsave.com +mamaslatinas.com +hstpnetwork.com +newsprints.co.uk +realclearworld.com +atpanel.com +ctx.ly +textme-app.com +conversantmedia.com +komikoyunlar.net +wimp.com +localyokelmedia.com +dateandtimesync.com +collider.com +clevergirlscollective.com +golferstrust.com +abebooks.co.uk +maskonline.vn +eff.org +visionobjects.com +xapads.com +noktamedya.com +mediafiredev.com +digitalinsight.com +mysoluto.com +vatgia.vn +jabong.com +ma.tt +emediate.se +shutterfly.com +shoppop.net +qz.com +appscloudupdater.com +adnexio.com +ykimg.com +terra.com.mx +popmyads.com +xat.com +mixi.jp +wefi.com +dtcn.com +cinesport.com +xatech.com +biography.com +k9webprotection.com +vmmpxl.com +uuidcshmg.com +bittorrent.am +arenajunkies.com +itar-tass.com +withoutabox.com +agoop.net +adorika.net +protectfootballonfreetv.com +msparktrk.com +ehowenespanol.com +ad2games.com +bloxcms.com +staplescenter.com +arcadefrontier.com +btg360.com.br +feedblitz.com +healthforself.com +yonhapnews.co.kr +tnaflix.com +tumra.com +veedi.com +taps.io +expedia.co.in +youwincdn.com +raiders.com +bet365affiliates.com +gci.net +sokrati.com +nordstrom.com +efinancialnews.com +freenode.net +projectwonderful.com +instinctiveads.com +thinkprogress.org +clickbooth.com +usaa.com +ddmcdn.com +macrumors.com +rmncdn.com +sublimevideo.net +predictad.com +megaoferta.net +kowalskypage.com +totallyher.com +appflood.com +startribune.com +yellowpages.ca +telesec.de +rstyle.me +scambioetico.org +komikler.com +theonion.com +tradelab.fr +gdmdigital.com +loudtalks.com +omiga-plus.com +doisongphapluat.com +mercadopago.com.br +plo.vn +danarimedia.com +ventunotech.com +adhispanic.com +tv.com +infusionsoft.com +besthitsnow.com +pub-fit.net +fusepowered.com +suprbay.org +32d1d3b9c.se +hellomagazine.com +rdcpix.com +trt.net.tr +lolking.net +edmunds.com +moodle.org +mercadoshops.com.br +exitmonetization.com +webme.com +c-col.com +livesportmedia.eu +wooga.com +hotwire.com +bit-search.com +localnet.com +123rf.com +highcharts.com +dashlane.com +chrysler.com +posst.co +meetrics.net +youjizz.com +warnerbros.com +bugsnag.com +stack.com +redirectingat.com +rightinthebox.com +copacet.com +timeapi.org +oyunkolu.com +getvideostream.com +cloudmark.com +lobstertube.com +maxiget.com +servetags.com +wp.pl +starwebnet.com +epa.gov +maturetube.com +cosmopolitan.com +mcproton.com +apuslauncher.com +eccmp.com +xdealvn.com +tvline.com +lostlettermen.com +free-porn-vidz.com +rasmussenreports.com +olivebrandresponse.com +lolboom.net +adfront.org +ajiang.net +inca.gov.br +popaholic.me +broadage.com +biphysics.com +devicevm.com +highbeam.com +giantbomb.com +lifegooroo.com +smartasset.com +9gaginc.com +history.com +crackedcdn.com +lithium.com +stagram.com +venere.com +redvertisment.com +cdn77.net +sedoparking.com +clickcountr.com +mediakit.com.br +widdit.com +sd-assets.com +pluso.ru +azcentral.com +webtretho.com +alice.it +webhostoid.com +orkut.com +helloreverb.com +healthline.com +belugaboost.com +suddenlink.net +2sawbucks.com +system-monitor.com +shelterpetproject.org +3gl.net +sgn.com +google.co.za +linkfeed.org +snacktools.net +geocities.com +pcgamer.com +diadiem.com +agoramedia.com +right-coupon.com +upstats.ru +hispeedtube.com +thinkfurtheralger.com +screencast.com +bna.com +nfl.biz +ashleyrnadison.com +tienphong.vn +bigpond.com +mazdausa.com +link.vn +dequeamaze.com +phys.org +openx.com +adzcore.com +desert-operations.com.tr +nflplayercare.com +teamspeak.com +thehindu.com +viewpoint.com +priceline.com +9game.com +finebooksmagazine.com +ttnetmuzik.com.tr +audible.co.uk +mercadopago.com +site-analytics.info +hao123.com.br +cwfservice.net +iconfinder.com +fjcdn.com +allyes.com +xmlshop.biz +flipora.com +afiliados.com.br +adnetwork.net +mefeedia.com +playblasteroids.com +cmbilisim.com +researchgate.net +mshcdn.com +tubemate.net +mobilepassback.com +moneycontrol.com +networkedblogs.com +adspdbl.com +shopclues.com +buzznet.com +canliskor.com.tr +adservlite.com +scientificamerican.com +sbbanner.com +drupal.org +babble.com +dailydot.com +120sports.com +expedia.es +phimmoi.net +fegn.com +bitfalcon.tv +cogmatch.net +marthastewart.com +peoplepets.com +fbnewsreport.com +futbolmacozetleri.com +bluewin.ch +wsjdigital.com +vidto.me +hindustantimes.com +cloud-trax.com +retailmenot.com +ibsrv.net +coed.com +sscdn.co +jobvite.com +imore.com +vagalume.com +fotokritik.com +usopen.org +giaoduc.net.vn +goviral-content.com +mkt932.com +details.com +realvu.com +wwv4ez0n.com +rapor.mobi +mymotocast.com +logly.co.jp +lolpro.com +jetlore.com +omnitwig.com +nakamitech.de +bizible.com +51y5.com +turbobytes.net +myfreecams.com +ensonhaber.com +thefiscaltimes.com +net-mine.com +torrent-download.to +adsupply.com +superantispyware.com +cinergroup.com.tr +systemcdn.net +melonstube.com +libsyn.com +tzoo-img.com +nixcdn.com +pch.com +globalenerji.com.tr +taptica.com +justlook.tv +tracksitetraffic1.com +maponics.com +truste.org +integral-marketing.com +comodo.net +messagingengine.com +sphinn.com +mackeeper.com +cpmrocket.com +blush.com +web.tv +id.net +guildwars2guru.com +gomtv.com +softonic.com.br +starzone.info +listenlive.co +v4cdn.net +dogusyayingrubu.com.tr +yontoo.com +dealchicken.com +filmon.com +news.com.au +tadst.com +bgov.com +ani-view.com +wnsqzonebk.com +mediashakers.tv +cazamba.com +dpreview.co.uk +xda-developers.com +space.com +hupso.com +djreprints.com +gelocal.it +khanacademy.org +stocktwits.com +minhngoc.net.vn +hud.gov +hairenvy.com +featurelink.com +arabayarisi.com.tr +williamhill.com +sportsnetwork.com +yandex.kz +sciencedaily.com +dolimg.com +komiksurat.com +technologytell.com +dribbble.com +iconarchive.com +cbscorporation.com +tinmoi.vn +adzhub.com +highwebmedia.com +fssta.com +sabah.de +impawards.com +passport.com +adohana.com +nflonlocation.com +globovideos.com +emailretargeting.com +webservis.gen.tr +webroot.com +qwikbookprint.com +magicfinds.com +maximustube.com +googleadsserving.cn +extensionanalytics.net +reddollars.com +browsersecurity.net +adelement.com +bacdau.vn +soundandglory.com +akhbarak.net +clorox.com +csnstores.com +springboardvideo.com +supremetube.com +wowdb.com +campanja.com +bettycrocker.com +usatodaysportsevents.com +anime-news.info +wbmd.com +verizoninsider.com +spongecellmedia.com +bbcworldwide.com +kmart.com +olx.com.br +thisamericanlife.org +banzaiadv.it +redbox.com +online.sh.cn +redstate.com +vogue.com.tr +compey.net +suddenlinkmail.com +hastrk3.com +orbitz.com +villas.com +imdbweb.info +aeerdy.com +expedia.co.nz +groupon.de +oprah.com +grouponworks.com +createspace.co.uk +faceporn.com +legalnotice.org +kaptcha.com +concentric.com +csnimages.com +reevoo.com +lacivertdergi.com +ookla.com +beekee-akkie.com +refdesk.com +downloadmeteoroids.com +superbahisaffiliates.com +palmcoastd.com +dizi-izle.com +luttgenheinrich.bz +sina.com +formstack.com +salemwebnetwork.com +livehelpnow.net +chitika.com +qualys.com +hostgator.com +sputnikhome.com +firmarehberiekle.gen.tr +indiebound.org +smartlifeweekly.com +xkcd.com +cpmfun.com +kompas.com +newsbusters.org +buysafe.com +slickdealscdn.com +ecustomeropinions.com +opaltelecom.net +cboeoptionshub.com +necn.com +volusion.com +kuaibo.com +gionee.com +zypush.com +cityads.ru +docstoccdn.com +habrahabr.ru +markmost.com +soso.com +diablofans.com +google.com.pr +trustsign.com.br +curseforge.com +tweetmeme.com +magnetmail1.net +target.ca +samsungalways.com +littlethings.com +bloombergsports.com +avg.cz +gazeta.pl +webtrackerplus.com +member-hsbc-group.com +craigslist.ca +nlinevideos.com +ansa.it +utsandiego.com +100im.info +torrent-downloads.to +emule.org.cn +inttrax.com +bloombergbriefs.com +faceporn.no +sec.gov +mirmay.com +zamunda.net +batanga.com +odatv.com +watchseries.lt +gotomeeting.com +fusion.net +in.com +ad-m.asia +playbuzz.com +wufoo.com +irctc.co.in +wpthemes.co.nz +playfizz.com +lossip.com +torrentreactor.net +imptrkr.com +bshare.cn +swagbucks.com +socialvi.be +opselect.com +fotolia.com +re-markable.net +is.gd +yenimedya.com.tr +cookingchanneltv.com +nflplayers.com +carambo.la +i-em.eu +letv.com +infoaxe.com +nbclearn.com +mint.com +porn.com +antiwar.com +fbiz.com.br +okccdn.com +oferta.vc +acuityads.com +nextperformance.com +torrentfreak.com +brightroll.com +krishnna.com +webhosteo.com +saoonline.vn +computerandvideogames.com +adshostiso.com +postdirect.com +audtd.com +recruitics.com +yelp.co.uk +ruten.com.tw +meridiana.it +traidnt.net +flixcar.com +ilfattoquotidiano.it +techepoch.com +foxitservice.com +lg.com +bestofmedia.com +drivergenius.com +dfdd4c0913aa193a3dd3d20b7645e2a46a3e4.com +silvercdn.com +scopely.io +o2.co.uk +cagesideseats.com +pegi.info +intagme.com +livedoor.com +google.ae +researchadvanced.com +india.com +khon2.com +move.com +who.is +peacockproductions.tv +techz.vn +mydotcomrade.com +famefocus.com +level3.net +agoda.net +lefigaro.fr +astromendabarand.com +digitalrivercontent.net +thedianerehmshow.org +kalooga.com +androidpolice.com +cvent.com +jossandmain.com +mediabistro.com +bdupdater.com +afp.com +bettermedicine.com +olivesoftware.com +rakuten.co.jp +vocativ.com +tnt-ea.com +cstv.com +hscta.net +theregister.co.uk +goo.ne.jp +deviantart.com +elle.com +contactlab.it +appdynamics.com +eurogamer.net +newtention.net +free-analytics.com +y8.com +bac.com +dangerousminds.net +softonic.it +umass.edu +demandmedia.com +joygame.com +tapulous.com +bookmyshow.com +71i.de +avito.ru +mxcdn.net +fpsgeneral.com +analytics-egain.com +fiverr.com +incitemedialabs.com +breakingnews.com +pubt.net +independent.ie +kbb.com +wptavern.com +9k.com.vn +vcdn.vn +sub.ly +rantchic.com +aionarmory.com +parenttoolkit.com +tripstodiscover.com +minecraftwiki.net +urbanspoon.com +ouedkniss.com +haberturk.tv +moreover.com +b117f8da23446a91387efea0e428392a.pl +woothemes.com +komikdunya.com +gw2db.com +onthemedia.org +umsns.com +outsports.com +yext.com +aruba.it +wetter.com +vividseats.com +helperbar.com +valuecpm.net +valvesoftware.com +oleane.net +fullhdfilmizle.org +oned.io +mercadolibre.com.ar +hsforms.net +wonderhit.com +virustotal.com +windowscentral.com +gmads.net +disneystore.com +takvim.com.tr +addmefast.com +chotot.vn +nifty.com +rbc.ru +carbonite.com +directv.com +octoshape.eu +command.com +grouponaffiliate.com +hpeprint.com +bodybuilding.com +pxxtz.com +amd.com +rollcall.com +mgm.gov.tr +imonomy.com +retargeter.com +socialgamenet.com +mdctrail.com +daum.net +maxwebsearch.com +itao.com +sittercity.com +nflevolution.com +fatakat.com +webmasterplan.com +onet.pl +twoo.com +v1cdn.net +comodoca3.com +ultradns.org +registeredsite.com +kontextua.com +submarino.com.br +infobae.com +souq.com +mcent.com +traffic-orgy.com +rzone.de +zeroredirect1.com +contentclick.co.uk +loginradius.com +kamcord.com +zeti.com +3366app.com +spinmedia.com +livenation.com +meme.vn +heise.de +ultradns.net +amazonbrowserapp.com +teleborsa.it +azurewebsites.net +baidu.com.br +download-servers.com +ultradns.biz +yarpp.org +nieonline.com +googlepages.com +chcmkt.com +costco.com +tubecup.com +darthhater.com +pptv.com +landsend.com +softonic.fr +btinternet.com +jcpenney.com +sephora.com +mndigital.com +dodge.com +walkscore.com +mobilenations.com +seznam.cz +ultradns.info +waterfrontmedia.com +interia.pl +etrade.com +radiolab.org +propellerpops.com +yelp.ch +cooladata.com +scansafe.net +tilt.com +atlasobscura.com +city-data.com +spoton.it +demonoid.ph +mediatakeout.com +simpsons-ea.com +fon.com +spot.im +compuwareapmaas.com +kboing.com.br +iqzone.com +eluniversal.com.mx +defaulttab.com +bungie.net +blocket.se +vitruvianleads.com +polygon.com +bloomberg.net +bild.de +unicef.org +tagged.com +kraloyun.com +lemagram.com +tagcommander.com +dealply.com +kitcode.net +samsung.com.br +ucoz.net +dummies.com +zoho.com +syn-api.com +gioneemobile.net +blogtopsites.com +lendingtree.com +televisionfanatic.com +kursus-bahasa.com +brucelead.com +sunrise.am +illiweb.com +rj.gov.br +sbito.it +tripit.com +turunculevye.com +cdn-hotels.com +gbga.gi +datacaciques.com +jcrew.com +unileverprivacypolicy.com +videozview.com +diigo.com +leasewebcdn.com +yotpo.com +fungame.com.br +google.com.ec +tripcurator.com +a433.com +ptd.net +geeksquad.com +publicsuffix.org +ck101.com +ccm2.net +yelp.de +arcadeyum.com +corel.com +meetupstatic.com +nguoiduatin.vn +cinemablend.com +terrariaonline.com +wowhead.com +list-manage.com +rondavu.com +ceryxefw.com +mongoosemetrics.com +evolvingseo.com +thehitsusa.com +gamepedia.com +crunchyroll.com +nbclosangeles.com +vidcoin.com +leboncoin.fr +qwest.net +my.com +mysql.com +naukri.com +wisersaver.com +firstlook.org +salecycle.com +couponcamp.com +foreverceleb.com +bblr.me +newdatastatsserv.com +pages02.net +hyperpromote.com +buyt.in +zcache.com +verticalscope.com +softonic.de +backpage.com +cloudtrax.com +nava.vn +bentenoyunlari.org +post-gazette.com +adbabylon.com +yelp.be +preguntados.com +htctouch.com +investopedia.com +kmdisplay.com +trklnks.com +politifact.com +cuti.vn +copyscape.com +betburdaaffiliates.com +nuvid.com +olx.in +gslbjpmchase.com +talk4free.com +coxmail.com +appscomeon.com +fisglobal.com +angelfire.com +hiido.com +install-daddy.com +free.fr +dimml.io +softonic.cn +streameye.net +sapo.pt +dmoz.org +yelp.fr +primewire.ag +sexlog.com +wbur.org +hm.com +firebase.com +helloridwan.com +advertiseonabout.com +easybib.com +juicyceleb.com +about.me +midnightjs.net +seattletimes.com +r10.io +linkbucks.com +mnetads.net +groceryserver.com +forobeta.com +digitalwindow.com +xuite.net +gtmetrix.com +bigfootinteractive.com +facenama.com +c4tracking01.com +picmonkey.com +taleo.net +4sqi.net +soubarato.com.br +wsjstudent.com +yelp.com.hk +gapinc.com +clarovideo.com +thewrap.com +yelp.nl +mit.edu +magnetmail.net +zimbio.com +bestofmicro.com +nctcorp.vn +hughes.net +spider.ad +pornerbros.com +wow-europe.com +agentesevenoteatro.com.br +infogame.vn +exilepro.com +jmp9.com +nbcphiladelphia.com +nivi.vn +twittercounter.com +medyanetplayer.com +pubmed.gov +demonware.net +cudasvc.com +deejay.it +pr-cy.ru +distilnetworks.com +es.pn +realharborredirect.com +mail.mil +tifbs.net +distractify.com +zulilyinc.com +nps.gov +online-adnetwork.com +tabnak.ir +anv.bz +magazinkolik.com +filmizle.com.tr +keek.com +upcmail.net +arcamax.com +puckermob.com +craigslist.co.za +firedrive.com +lightinthebox.com +makemytrip.com +diretta.it +irs01.net +tiin.vn +moovweb.net +expedia.at +flixfacts.com +nextissue.com +classistatic.com +fifa.com +gyazo.com +google.com.do +homedecorators.com +nbcwashington.com +kmylvwo5.com +shazamid.com +skysports.com +gpm-digital.com +hdfcbank.com +welt.de +carfax.com +fhserve.com +mymovies.it +assineabril.com.br +redfin.com +m-w.com +expedia.com.my +free-tv-video-online.me +netease.com +web-18.com +scoop.it +zdassets.com +cnetfrance.fr +surveywriter.net +yelp.fi +p5w.net +voegol.com.br +eircom.net +puppytoob.com +yelp.ca +bolumsonucanavari.com +movie4k.to +vzwshop.com +newsday.com +superpages.com +bestblackhatforum.com +getsidekick.com +hespress.com +clocklink.com +farsnews.com +ahaber.com.tr +terra.cl +miniclippt.com +onlinesbi.com +expedia.be +nate.com +lululemon.com +epson.com +sc2mapster.com +tuenti.com +wowace.com +airtel.in +mercadolibre.com.mx +yelp.es +websitealive.com +blogspot.co.uk +abc.es +persianblog.ir +glanceguide.com +google.hr +altervista.org +cnetnews.com.cn +marriland.com +elance.com +samsungallstore.com +teacherspayteachers.com +cpatrendreklam.com +pravda.com.ua +searchforce.net +cam4.com +mobile.de +canada.com +dotki.tv +smarterpowerunite.com +adobe.io +metalyzer.com +walkme.com +justdial.com +cnetcontent.com +tistory.com +ifengimg.com +purenetworks.com +vivastreet.it +r-ad.ne.jp +coolots.com +theroot.com +wibiya.com +google.kz +semrush.com +tianya.cn +joystiq.com +quicknessrun.com +knight-sac-media.com +netindex.com +nickmom.com +58.com +kakaku.com +watchmygf.net +vennq.com +baixakijogos.com.br +dubizzle.com +firstpost.com +brilliantearth.com +csnbayarea.com +dellbackupandrecoverycloudstorage.com +cloneweb.net +zqlx.com +douban.com +aparat.com +thesportster.com +odesk.com +idnes.cz +tagstat.com +myntra.com +thesun.co.uk +evitecdn.com +phonearena.com +aizhan.com +a2dfp.net +hit.ua +anddownthestretchtheycome.com +yelp.com.au +adapf.com +tabelog.com +ijreview.com +37signals.com +dealer.com +dailynews.com +abine.com +tim.it +flix360.com +pingtest.net +rotowire.com +storm8.com +uribl.com +motthegioi.vn +lanacion.com.ar +staplesadvantage.com +nouvelobs.com +vesti.ru +wwe.com +horoscopedays.com +rovicorp.com +ltn.com.tw +premiumtv.co.uk +icicibank.com +2345.com +intoday.in +sex.com +cdntraffic.com +mihanblog.com +rightmove.co.uk +komiksozler.net +aniways.com +bravotube.net +impressiondesk.com +abt.cm +neobux.com +georiot.co +majesticseo.com +memurlar.net +installerapplicationusa.com +bankmellat.ir +sophosupd.net +magentocommerce.com +vtexrc.com.br +sueddeutsche.de +cvs.com +expedia.com.br +huffingtonpost.it +ione.net +qianlong.com +tiny.cc +appointron.com +tapit.com +almasryalyoum.com +leo.org +pchome.com.tw +globoesporte.com +app111.com +powermarketing.com +yelp.com.br +playtopus.com +51fanli.com +autohome.com.cn +lequipe.fr +jeuxvideo.com +aplus.com +firsttoknow.com +military.com +yelp.it +feelcars.com +cyberlink.com +gelirortaklari.com +novinky.cz +premierleague.com +ingresso.com +jagran.com +x17online.com +xadcentral.com +cumulus-cloud.com +nownews.com +dpliveupdate.com +ccb.com +dmm.com +startpage.com +gameinformer.com +huyenbi.net +herewetest.com +podiumcafe.com +cnmo.com +gome.com.cn +airsensewireless.com +softonic.jp +g8teway.com +wpmudev.org +2ch.net +gumtree.com +ku6.com +paipai.com +rednet.cn +purebreak.com.br +800wen.com +condenet.com +gsmarena.com +pacsun.com +businessinsider.com.au +hotmail.com.br +pbskids.org +gismeteo.ru +nairaland.com +hqq.tv +yelp.co.nz +google.cn +searchengines.ru +yelp.com.ar +freelancer.com +chaseswing.eu +prothom-alo.com +livingplay.com +mkt922.com +immobilienscout24.de +smartmoney.com +e-printphoto.co.uk +eventoptimize.com +shaadi.com +templatemonster.com +internetbrands.com +yelp.at +google.lk +pixiv.net +inventorycreation.com +google.rs +pingdom.com +prestashop.com +oneindia.in +payoneer.com +r10.net +reverso.net +yelp.com.sg +empowernetwork.com +fullscreenweather.com +paginegialle.it +wowpedia.org +sosmart.vn +ce.cn +kariyer.net +zol.com.cn +mywebsearch.com +google.az +freep.com +wsj.com.tr +lge.com +yelp.co.jp +mystart.com +cnet.de +appledaily.com.tw +blogtalkradio.com +computerworld.com +apartmenttherapy.com +shopstyle.com +chip.de +px10.net +hi-spider.com +softonic.pl +popmog.com +timeout.com +bitauto.com +adne.tv +google.com.ly +people.com.cn +gazzettaobjects.it +10best.com +megacurioso.com.br +brandreachsys.com +pchome.net +symphonytools.com +kankan.com +clixsense.com +guardianapis.com +narod.ru +probux.com +qtrax.com +adsbackup.net +it168.com +americanlivewire.com +forgeofempires.com +onlylady.com +consumerreports.org +growmobile.com +blogfa.com +wcpo.com +sberbank.ru +resultsaccelerator.net +google.com.kw +citicards.com +mx25.net +bing4.com +fanfiction.net +directadvert.ru +flixster.com +ileehoo.com +vezuha.me +hdfilmsitesi.com +stridenation.com +starbaby.cn +newsgator.com +ioladv.it +chinatimes.com +sfglobe.com +yelp.cl +zippyshare.com +csdn.net +roblox.com +elegantthemes.com +adserving.jp +xgo.com.cn +gazeta.ru +e-junkie.com +fdlstatic.com +blogspot.com.tr +homeaway.com +icast.cn +yelp.ie +allmyvideos.net +appisys.com +sociablelabs.com +youth.cn +orf.at +sitepoint.com +webmoney.ru +allocine.fr +uclick.com +yesky.com +blogspot.jp +gigacircle.com +google.com.ng +hupu.com +mercadolibre.com.ve +jrj.com.cn +lds.org +sulekha.com +varzesh3.com +jvzoo.com +diceholdingsinc.com +jimdo.com +h12-media.com +ashford.edu +viss.vn +gresille.org +xvika.com +blogspot.gr +etao.com +google.com.pk +tokobagus.com +lancenet.com.br +arcadeparlor.com +cj.com +psychologytoday.com +incapsula.com +gmw.cn +youyuan.com +blogspot.in +gutefrage.net +yoka.com +haiwainet.cn +hatena.ne.jp +indiamart.com +tiny-toyz.com +sunporno.com +blogspot.de +evbuc.com +blackberry.net +iplt20.com +sape.ru +tructiepbongda.com +theskipshot.com +blogspot.com.ar +wideinfo.org +epicurious.com +blogspot.ru +chatidcdn.com +epimg.net +kdnet.net +voc.com.cn +trovigo.com +guzelleselim.com +hudong.com +nzn.me +atvavrupa.tv +eastday.com +google.com.bd +prismamediadigital.com +over-blog.com +plaintube.com +gr.pn +opensiteexplorer.org +tractionize.com +nationalgeographic.it +b5m.com +gamerankings.com +tmzstore.com +acesse.com +china.com +markafoni.com +url.cn +lnkdatas.com +comenity.net +qone8.com +blogspot.com.br +mpnrs.com +insnw.net +sgnapps.com +viadeo.com +dailysabah.com +pixnet.net +vodtraffic.com +ajc.com +tukif.com +xpopad.com +123srv.com +matchflowmedia.com +chexun.com +sakura.ne.jp +yelp.com.mx +ca.gov +hpdjjs.com +nicovideo.jp +bigdoor.com +vtex.com.br +zond.org +focus.de +life.com.tw +systemmonitor.us +39.net +delivery51.com +pcgames.com.cn +convio.net +thefreecamsecret.com +wildstarforums.com +cdnst.net +blogspot.mx +dainikbhaskar.com +seat.it +o24x7.com +blackhatworld.com +petflow.com +skyrimforge.com +lady8844.com +mama.cn +dol.gov +gamned.com +ameba.jp +bigfoot.net +seesaa.net +voanews.com +ccloud.io +walmartlabs.com +eazel.com +getaviate.com +noip.com +targetphoto.com +showtv.com.tr +mysearchresults.com +behindthesteelcurtain.com +quovadisglobal.com +civicplus.com +xcar.com.cn +ettoday.net +gateable.com +stockstar.com +baomihua.com +blogspot.com.es +srv123.com +tvdata.com.br +staples-3p.com +rw.gs +pconline.com.cn +warriorforum.com +clicrbs.com.br +nonstoppartner.net +kinopoisk.ru +yelp.com.tr +neemu.com +genieo.com +pengyou.com +dmm.co.jp +weloveiconfonts.com +kym-cdn.com +m2newmedia.com +linkszb.com +cntv.cn +reallifecam.com +softlayer.net +mmbang.com +uctrac.com +commentcamarche.net +huff.lv +revistamonet.com.br +pbsrc.com +scholastic.com +soku.com +buy-targeted-traffic.com +17ok.com +tim.com.br +wmnlife.com +homedepot.ca +clickbank.com +fuckish.com +v1.cn +4399.com +asos.com +eyny.com +starmagazine.com +baofeng.net +superstoragemy.org +ucoz.ru +imgaft.com +instair.net +sharelive.net +abebooks.it +sap.com +theadex.com +m-decision.com +depositfiles.com +yourtango.com +bkstr.com +hswstatic.com +avantlink.com +dailysanctuary.com +haber-sistemi.com +webhostingtalk.com +hitfix.com +reachmax.cn +admatic.com.tr +purch.com +yelp.dk +kundenserver.de +bookbub.com +sfdcstatic.com +caijing.com.cn +glamour.com +giadinhonline.vn +register.com +enet.com.cn +kaskus.co.id +adsniper.ru +winzip.com +allmovie.com +myshopify.com +lync.com +webs.com +loopnet.com +chinaz.com +awesomehp.com +adreactor.com +geek.com +mbc.net +chatid.com +vuze.com +b2wdigital.com +gamesradar.com +aejohg.com +womenshealthmag.com +brasilescola.com +pcbaby.com.cn +shopkrowd.com +eddie4.nl +jqw.com +expedia.co.id +yelp.cz +yelp.se +yaolan.com +reason.com +homedepot.com.mx +bricknet.com +lgcpm.com +craigslist.com.ph +weightwatchers.com +jw.org +tribpub.com +yelp.pl +plaxo.com +requestnextadnet.com +traileraddict.com +adtrustmedia.com +lga.org.mt +barbioyunlari.org +zybez.net +moceanads.com +kopimi.com +adiquity.com +bleedinggreennation.com +vietnamnetad.vn +biglobe.ne.jp +expedia.com.hk +staticamzn.com +freeserve.com +csnchicago.com +rentedspaces.com +newscientist.com +redbeacon.com +adlabs.ru +kitchenstoringshop.com +cpmbux.com +anythumb.com +cpmaxads.com +iddaa-siteleri.com +t-online.de +uber.com +supercounters.com +leylek.com +baotintuc.vn +dafont.com +mobile01.com +path.com +hypergames.net +toplist.sk +appnexus.com +easports.com +forbeschina.com +reverbnation.com +blogglez.com +shopyourway.com +dlvr.it +live-genieo-feed.com +imo.im +tbliab.net +asana.com +mnn.com +thehollywoodmag.com +daohongdonvenus.com +forbes.com.tr +trinklink.com +mobdub.com +cam4ads.com +craigslist.co.in +bol.com.br +browsersafeguard.com +sesamestats.com +ticketmaster.co.uk +tmztour.com +rek.mobi +weknowmemes.com +fark.com +mq4m.com +cheaptickets.com +hot-cpm.com +adplxmd.com +nimbuzz.com +commerce.gov +ad.org.vn +impactradius.com +jotform.com +forbesmagazine.com +furl.net +dnaindia.com +videoentertainmnt.com +caferuj.com.tr +bnef.com +docer.com +nu.nl +eater.com +liftoff.io +samsungchaton.com +gulfup.com +asus.com.tw +aboneturkuvaz.com +hitslink.com +themetapicture.com +funnyordie.com +htkulup.com +avazudsp.net +dvdcdn.com +vivox.com +ihg.com +alljsscript.com +advolution.de +yelp.no +admixer.net +configar.org +citizenjournal.net +sacbee.com +fingersoft.net +wrating.com +rsys2.net +set.tv +list.ru +comicvine.com +eb.com +expedia.com.ar +kooora.com +utop.it +masrawy.com +1up.com +moneycontrol.co.in +hongkiat.com +beytoote.com +securedatatransit.com +network-auth.com +harvard.edu +filseclab.com +ikikisilikoyunlar.com +mixcloud.com +openadserve.com +aeon.co +janrainsso.com +bebegimvebiz.com.tr +interesticle.com +yelp.pt +chzbgr.com +zello.com +cams.com +exchangedefender.com +horyzon-media.com +boredbug.com +craigslist.de +forbes.com.mx +jscount.com +yisou.com +onlinehome-server.info +musicbrainz.org +semantictec.com +newsdev.net +haivainoi.com +thoughtcatalog.com +dnainfo.com +extremetech.com +layered.net +magnumads.me +starwoodhotels.com +innityserve.net +superiends.org +zapto.org +craigslist.com.tr +cosmodergi.com +forever21.com +moviepilot.com +scmpacdn.com +screencrush.com +chicos.com +depend.com +travelocity.com +walmart.ca +huluad.com +tokenads.com +craigslist.com.sg +gsfn.us +mendeley.com +tapsense.com +domaintools.com +expedia.dk +twc.com +nowvideo.sx +soclminer.com.br +t.cn +irrawaddy.org +yourbridebook.com +el-mundo.net +isbank.com.tr +oyunlar1.com +oyunvitrini.com +cosmogirl.com.tr +cdw.com +scrippsnationalnews.com +mobidea.com +logme.in +domainsponsor.com +tipo777.com +theline.com +pandawhale.com +zdworks.com +webgains.com +triongames.com +iht.com +swamedia.com +trumba.com +craigslist.com.tw +republer.com +adskyforever.com +craigslist.at +casaevideo.com.br +clip.vn +bangkokpost.com +craigslist.jp +craigslist.fr +d4p.net +pastaoyunu.com +esquire.com.tr +billdesk.com +livepcsupport.com +dhgiris.com +craigslist.be +simplyhired.com +ad122m.com +linternaute.com +mkt941.com +haivlfan.com +hubimg.com +rncdn1.com +thelocalsearchnetwork.com +knoworthy.com +bseller.com.br +peoplepc.com +makazi.com +ics0.com +mindbodyonline.com +tube911.com +minika.com.tr +vporn.com +cia.gov +clicksvenue.com +craigslist.dk +craigslist.com.cn +axs.com +guiamais.com.br +kanimg.com +silence-ads.com +tmztournyc.com +craigslist.gr +fox40.com +rummblelabs.com +jassets.com +glotorrents.com +craigslist.fi +claromusica.com +expedia.co.kr +battlefield.com +nationalpost.com +bloomberglaw.com +gamer.com.tw +cincyjungle.com +spokeo.com +megabrowse.biz +iphmx.com +reignofgaming.net +rexposta.com.br +admnx.com +mikle.com +cybertrade.co.za +craigslist.pl +inspcloud.com +uzjvh.com +joblo.com +redbull.com +securitymetrics.com +teknokulis.com +spilcloud.com +gizmodo.es +elwatannews.com +isteinsan.com.tr +pmc.com +minhaserie.com.br +expediamail.com +dt00.net +nhle.com +reinvigorate.net +christianbook.com +zerohedge.com +20minutos.es +hsbc.com.br +craigslist.it +9v8kxvfvw.com +flipagram.com +dynect.net +conduit-data.com +expedia.fi +meus5minutos.com.br +craigslist.pt +torrent-finder.info +incredibarvuz1.com +theepochtimes.com +logos.com +otohaber.com.tr +nginx.org +telenet-ops.be +snapfish.com +hub.am +slashdotmedia.com +craigslist.es +mmtro.com +ehow.com.br +newsfactor.us +craigslist.se +votinginfoproject.org +726.com +diynetwork.com +nguyenkim.com +wps.cn +softpedia.com +onforb.es +minq.com +vmware.com +ink1001.com +mysearch-online.com +shopsocially.com +applift.com +dict.cc +hsn.com +enigmasoftware.com +beatsmusic.com +cifraclub.com.br +oglobo.com.br +info-stream.net +arrowheadpride.com +variety411.com +teddybrinkofski.com +piclens.com +un.org +sanoma.fi +jobrapido.com +gruponzn.com +aka.ms +live-lyrics.com +craigslist.co.uk +idealo.de +expedia.ie +resellerratings.com +epi.vn +cp20.com +online.de +travelchannel.com +crackberry.com +reachjunction.com +putlocker.bz +ovh.net +mediabong.com +caspion.com +paddypower.com +verizonbusiness.com +aftonbladet.se +timesofindia.com +qcloud.com +objectedge.com +rcsmediagroup.it +fun.tv +greatarcadehits.com +reliableremodeler.ca +admoda.com +widespace.com +cymera.com +baltimorebeatdown.com +realtracker.com +baomoi.mobi +glu.com +cy-pr.com +fmsads.com +cttsrv.com +samsungadhub.com +dafity.com.br +digikala.com +magreprints.com +hitsprocessor.com +pampanetwork.com +twnmm.com +leguide.com +whitehouseblackmarket.com +expedia.nl +webmdhealthservices.com +truyenhinhanvien.vn +woobox.com +easy2.com +publicradio.org +mtvnimages.com +forbes.pl +craigslist.ch +rockstargames.com +afar.com +ramtrucks.com +tenethealth.com +investingchannel.com +gettvwizard.com +wnco.com +condenaststore.com +digitalspy.co.uk +strava.com +comprises.info +paradergi.com.tr +nordstromimage.com +micromaxinfo.com +rocketfuel.com +bigblueview.com +angelpush.com +liveperson.com +skyhookwireless.com +haizap.com +openadserving.com +worldweatheronline.com +yikyakapi.net +justice.gov +craigslist.com +milanuncios.com +radyoturkuvaz.com +qzone.com +imagebam.com +usasabah.com +evmanya.com +refinedads.com +storycorps.org +battleredblog.com +bangmygfs.com +expedia.com.ph +expedia.co.th +neweggflash.com +nick.com +snmmd.nl +scottrade.com +onpointradio.org +latinvestor.com +bedbathandbeyond.com +domainnamesales.com +zmags.com +mozilla-europe.org +springer.com +vinepair.com +akismet.com +ccgslb.com +publy.net +chemistrychef.com +msrch.com +glide.me +hrw.org +uni-rostock.de +wnd.com +intuitcdn.net +realist.gen.tr +yahoo.com.br +onlymyhealth.com +china.com.cn +transmissionbt.com +huanqiu.com +dyngate.com +acmepackingcompany.com +buffalorumblings.com +coronalabs.com +tinthethao.com.vn +gingersoftware.com +geico.com +miniinthebox.com +bitcomet.com +nuance.com +atvnetworks.tv +yougov.com +ccmbg.com +mckesson.com +hearstdigital.com +phunutoday.vn +icims.com +curiyo.com +magicjack.com +sofra.com.tr +technologyreview.com +tmzhollywoodsports.com +superbahis217.com +cequinttmoecid.com +cheezburger.com +ci123.com +adrtr.net +informer.com +truefitcorp.com +pnc.com +powerlinks.com +craigslist.com.au +csidata.com +iobconcursos.com +homestead.com +rawstory.com +binaryoptionstm.com +expedia.com.sg +expedia.mx +pangia.biz +ad6.fr +drtvtracker.com +golfdigest.com +capitalone360.com +apiodyth.com +commissariatodips.it +e-pages.dk +kia.com +nuancemobility.net +expedia.com.tw +samdan.com.tr +fuse.net +sciencemag.org +getpantheon.com +reali.st +trendyol.com +correioweb.com.br +houselogic.com +streambroadcastmedia.com +hexagon.cc +bigcatcountry.com +thoigian.com.vn +forbesid.com +futurenet.com +ekstat.com +trustgo.com +alarabiya.net +surface.com +shoefitr.com +biznessapps.com +digitalfirstmedia.com +gopro.com +webaslan.com +haberzamani.com +adstatic.com +jointheteam.com +dailyinfovideo.com +bkatjs.info +money.com +vgsgaming-ads.com +bbm.com +photorank.me +ptreklamcrv.com.tr +observer.com +autotrendworld.com +cookfor1.com +t411.me +upsieutoc.com +timehop.com +expedia.no +isprimecdn.com +contentexplorer.net +da-ads.com +tipki.it +turkuvazmobil.com +sezgisel.com +admeme.net +bloombergtradebook.com +clickcarreira.com.br +scoringserving.net +trangvangvietnam.com +dose.com +guvenliinternet.org +inkfrog.com +rivalo3.com +placed.com +metacafe.com +datingvip.com +loseit.com +bloomberglink.com +infashionmag.com +dsnetwb.com +bloggingtheboys.com +backcountry.com +adtlgc.com +trovit.com +vechai.info +dallasnews.com +craigslist.com.pe +vote411.org +anonym.to +afip.gob.ar +instacam.com +codeplex.com +dailynorseman.com +schoolwires.com +craigslist.com.mx +shappify.com +boltsfromtheblue.com +oldnavy.com +vidprocess.com +markedup.com +ampagency.com +hoverzoom.net +hbo.com +sandclowd.com +womenpov.com +rtbpopd.com +joomlatune.com +msft.net +torrenty.org +altincicadde.com +hsappstatic.net +vef.vn +zndsk.com +amazon-press.it +nowvideo.at +gdgt.com +binary.net +lyonnaise-des-eaux.fr +creditkarma.com +splashnewsonline.com +zassets.com +pub-fit.com +mnectar.com +yeniasir.com.tr +ishort.co +research-int.se +turkuvazabone.com +zapkolik.com +forbesindia.com +dawgsbynature.com +navisite.net +shinobi.jp +adkengage.com +backup.com +milehighreport.com +incehesap.com +novanetservice.com +nld.com.vn +411.com +i-vietnam.vn +mystreamservice.com +cdnslate.com +piratebaytorrents.info +shoptime.com.br +style.com +rapidshare.com +buyatoyota.com +shopperapproved.com +mentalfloss.com +ninersnation.com +newinputinfoservice.com +kohlscorporation.com +bucsnation.com +bannersnack.com +frogupdate.com +futurecdn.net +phluant.com +itc.cn +pronto.com +newstogram.com +canalstreetchronicles.com +asda.com +modernluxury.com +usat.ly +happytrips.com +socialbeauty.com.br +rising.com.cn +watchguard.com +turkuvazmatbaacilik.com +helioscloud.com +bongdep.com +fap.to +unibet.com +joygamedl.com +ddccdn.com +bresnan.net +expedia.se +nextag.com +netshoes.com.ar +steepto.com +gamcare.org.uk +muare.vn +trust-guard.com +eye.fi +ibt.com +gamib.com +businessinsider.in +givemesport.com +mydomain.com +shar.es +seattlepi.com +o2online.de +wikia.net +s-analytics.info +conversionsbox.com +smi2.ru +tdameritrade.com +gigcount.com +ewebse.com +befrugal.com +fieldgulls.com +monografias.com +doodle.com +google.iq +catscratchreader.com +turkuvazyayin.com.tr +biddingx.com +atomz.com +webex.com +greenerweb.info +clker.com +hc.ru +yapikredi.com.tr +streamsend.com +bizlive.vn +jivosite.com +crisppremium.com +telestream.net +lzjl.com +vimg.net +pathfinder.com +vodlocker.com +xahoi.com.vn +widgetserver.com +appsmartpush.com +registeridm.com +tradetracker.net +ekstrabladet.dk +hexagram.com +tagboard.com +newswhip.com +oranara.com +assets-gap.com +bongdainfo.com +vuiviet.vn +yeniaktuel.com.tr +wilink.com +angieslist.com +clktraker.com +echoplatform.com +airbnb.com +quality-channel.de +stampedeblue.com +islenogren.com +flagcounter.com +musiccitymiracles.com +cjsab.com +walmart.com.mx +ganggreennation.com +image-maps.com +servedby-buysellads.com +stbm.it +vietnamnettv.vn +jetblue.com +governoeletronico.gov.br +rebelmouse.com +laodong.com.vn +nflplayerengagement.com +brightcove.net +primusad.com +alphassl.com +roixdelivery.com +myfox8.com +expedia.com.vn +randomhouse.com +villagevoice.com +gospect.com +brainyquote.com +rollbar.com +napster.com +marketgid.com.ua +decompras.com +interstats.org +airmail.net +metoffice.gov.uk +betfair.com +vidcore.tv +rabbitscams.com +pvp.net +bloombergindexes.com +celticsblog.com +sonyericsson.com +decolar.com +ligtv.com.tr +staticloads.com +parents.com +samsclub.com +catho.com.br +citysearch.com +tictacti.com +rewardtv.com +detroitnews.com +dizibox.org +delivery55.com +daumcdn.net +xyimg.net +payzippy.com +ycharts.com +cartoontube.com +netshoes.com.mx +beead.co.uk +edmodo.com +adviator.com +1and1.co.uk +usatodayeducation.com +coupons.net +mediaset.net +burstbeacon.com +snoonet.org +remintrex.com +topeleven.com +realprotectedredirect.com +mixx.com +smiles.com.br +arcsoft.com +forbesmedia.com +wfp.org +look.io +gigaom.com +prideofdetroit.com +9c9media.com +vonage.net +mybinarysystem.com +edmunds-media.com +siteapps.com +gammae.com +bugherd.com +thekitchn.com +forbesglobalceoconference.com +kiwiirc.com +comcastspotlight.com +drudgereportarchives.com +www8-hp.com +cpmaffiliation.com +patspulpit.com +thephinsider.com +mail.dk +peoplem.ag +gutenberg.org +newsrep.net +pitchfork.com +spanishdict.com +yeniasirilan.com +spamexperts.net +vcommission.com +blueserving.com +dvipcdn.com +rutube.ru +hogshaven.com +mightytext.net +cdnplanet.com +bigpoint.com +informars.com +stubhub.co.uk +callofduty.com +dailyprofitmethod.org +p2pdl.com +gallup.com +cpc-ads.com +patheos.com +ultimedia.com +feeyun.com +fcounter.info +telia.com +jc-affiliates.com +optusnet.com.au +4cdn.org +dr.dk +fntk.co +mktw.net +go2jump.org +leonardo.it +synacast.com +gossipcop.com +canadapost.ca +planet.nl +stopbadware.org +sublimetext.com +cabbjs.info +pantherssl.com +zam.com +arxiv.org +icopyright.net +aegworldwide.com +healthcare.gov +matomy.com +wdtvlive.com +forbesrussia.ru +adicio.com +ventivmedia.com +bloombergsef.com +iodonna.it +archive.is +fundacioncarlosslim.org +maclife.com +vistaprint.com +ctt.ec +palocalworld.info +babycentre.co.uk +emediate.com.br +sky.fm +flashget.com +utarget.ru +purechat.com +videosense.com +jsonline.com +appstore.com +bannersnack.net +quizgroup.com +datawire.net +mensfitness.com +directads.de +chardward.us +lxdcdn.net +24h-hotel.com +silverandblackpride.com +markitcdn.com +schlund.de +speedbit.com +ad-stir.com +addlive.io +wt-eu02.net +mediasetpremium.it +nesn.com +myharmony.com +senzari.com +99widgets.com +dvs.vn +dummy-domain-do-not-change.com +therichest.com +watchmygf.com +cloudinsights.com +concursolutions.com +ehow.co.uk +ddni.net +uploadable.ch +kn3.net +findarticles.com +cleantechnica.com +revengeofthebirds.com +contactatonce.com +connectify.me +zamimg.com +5giay.vn +extra-imagens.com.br +wscdns.com +hrdepartment.com +netsdaily.com +thefalcoholic.com +gorillions.com +classicshell.net +zunnit.com +hoobly.com +globalmediaserving.com +namequery.com +wd2go.com +wolframalpha.com +csafer.net +bradesconetempresa.b.br +mbtrx.com +barilliance.net +coveritlive.com +memecdn.com +slingmedia.com +mailshop.co.uk +cobex.net +twitchy.com +patricinhaesperta.com.br +thedailyswarm.com +epi.com.vn +routerlogin.net +highspeedbackbone.net +thebiglead.com +dhhs.gov +cntrafficpro.com +libertyballers.com +showtvnet.com +noom.com +merchantadvantage.com +builddirect.com +strcst.net +itv.com +schwab.com +vads.net.vn +n111adserv.com +cdndelivery.com +pixenka.com +anywho.com +cumhuriyet.com.tr +medicarenoticedeal.me +surpax.net +creativecloud.com +fema.gov +sadecehosting.com +totaltech.it +malwarebytes.org +blekko.com +globalnews.ca +ntradmin.com +ole.com.ar +viddler.com +grupoabril.com.br +technobuffalo.com +air2s.com +payments-amazon.com +babiesrus.com +microsoftvirtualacademy.com +getsidecar.com +dpstatic.com +business-standard.com +conxport.com +techrepublic.com +gravityrd-services.com +cnappbox.com +pagesuite-professional.co.uk +behe.com +sms-mmm.com +7graus.com +samsungvideohub.com +mid-day.com +postingandtoasting.com +harpersbazaar.com +realclear.com +tripadvisor.it +staples-static.com +freenet.de +bodis.com +tempoagora.com.br +appbrain.com +16mm.it +sinemadafilmizle.com +totalfilm.com +extfeed.net +pctools.com +imgci.com +betbooaffiliates.com +rave-api.com +tvgcdn.net +windycitygridiron.com +akbank.com +connatix.com +key.com +iinet.net.au +kampyle.com +dnsalias.com +thefederalist.com +dice.com +arstechnica.net +homestore.com +013net.net +interlude.fm +meneame.net +quotemedia.com +myfreeyp.com +senate.gov +thoughtsondance.info +pornoid.com +pbc.com +shorte.st +pri.org +microsoftonline-p.net +1worldonline.com +yourdailyscoop.com +vneconomy.vn +daddymami.net +online.no +gamespy.com +groupon.it +wsj.de +hyperadslite.com +savefreescoresseekers.me +godatafeed.com +democracynow.org +adspirit.de +ad131m.com +bet.com +mic.com +chow.com +clubpenguin.com +expediaaffiliate.com +itsfogo.com +chacha.com +clearsale.com.br +eurosport.com +brighteroption.com +bls.gov +turfshowtimes.com +wmobjects.com.br +aufeminin.com +fromthetop.org +contactmusic.com +bitbucket.org +the9.com +midco.net +bodybuilderdaily.com +goseeklocation.com +redirecting.ws +express.co.uk +superdownloads.com.br +boxcloud.com +iinmobi.com +modamob.com +strands.com +rei.com +adtiger.de +boingtv.it +walmartcontacts.com +cbsstatic.com +adsynth.com +timberland.com +trackedlink.net +assinefolha.com.br +peer5.com +opbandit.com +vanguard.com +klippal.com +naturalmotion.com +irishcentral.com +offeredby.net +fedexsameday.com +gostats.com +iyuntian.com +sage.co.uk +migre.me +bznx.net +realtor.org +mydati.com +ibtimes.co.in +m2o.it +direct-tap.com +homedepotmeasures.com +mailconnected.co.uk +knowyourmeme.com +realclearscience.com +gtdaily.com +terra.com.ar +vitamio.org +ctia.org +expediainc.com +cerberusapp.com +name-services.com +cnnturk.com +onionstatic.com +ctv.ca +dota2wiki.com +dropboxstatic.com +pclncdn.com +bringyourchallenges.com +capptain.com +yimg.jp +qrius.me +humanevents.com +interingilizce.com +inskinad.com +tomsitpro.com +ziffprod.com +hotelurbano.com.br +berries.com +taobao.org +mobitv.com +bravotv.com +iba.com.br +amzn.com +igg.com +ordergroove.com +cnbcprime.com +voicestar.com +tamind.ir +pianetadonna.it +tns-gallup.dk +snip.ly +mediashopping.it +ctctcdn.com +findagrave.com +ieee.org +r29static.com +pncmc.com +qiyipic.com +prnx.net +recipe.com +networkanalytics.net +flip.it +investingmediasolutions.com +searchfun.in +bradescofinanciamentos.com.br +plex.bz +governmentjobs.com +usablenet.com +eathei.com +starfluff.com +jacobs.com +kul.vn +viki.com +premiereradio.net +4chan.org +jampp.com +twenga.com +nextopiasoftware.com +ugwdevice.net +sonic.com +poweroffer.net +snopes.com +thefreelibrary.com +fullsail.edu +shareyourlink.net +fivestore.it +supermaneddy.com +installerdatauk.info +mailroute.net +brainpop.com +raptorshq.com +hotwords.com +digitalpoint.com +d2hshop.com +seatguru.com +wnba.com +boxcdn.net +games724.com +mzcdn.com +mimicromax.com +viewmotions.com +wordreference.net +canlitv.tv +ifc.com +cdnjs.com +aspplayground.net +emagazines.com +abacast.net +bellaliant.net +squareup.com +ipgeoapi.com +bandsintown.com +xoedge.com +bizj.us +adtheorent.com +ecollege.com +vodafone.com +yr.no +envato.com +kaskus.com +1dial.com +glo.bo +revnm.com +bonappetit.com +api-alliance.com +nbcbayarea.com +adnotch.com +cotssl.net +irc.su +novalayer.org +southwestvacations.com +unbxdapi.com +celljournalist.com +sciencefriday.com +alloyentertainment.com +itaringa.net +drivershq.com +adsunflower.com +anysex.com +meride.tv +tinyco.com +sanalpazar.com +continular.com +ilsemedia.nl +wn.com +vetogate.com +appgenuine.com +freeonlineusers.com +jotfor.ms +indulgy.com +gizmodo.co.uk +syncaccess.net +filmesonlinegratis.net +matheranalytics.com +cloudcdn.net +mediaweek.com +8digits.com +systemmonitor.co.uk +creditoruralcaixa.com.br +sondakika.com +eaton.com +blogs.com +getrockerbox.com +mangarockhd.com +tintuconline.com.vn +cabinet-office.gov.uk +ccsp.com.br +manta-r2.com +myofferspro.com +atlas.com +livestrongcdn.com +camera360.com +buienalarm.nl +gamewall.me +wildtangent.com +ameritrade.com +kitco.com +cnzz.net +xoso.net +digilant.com +crackle.com +segpaycs.com +spartzmedia.com +naseej.com.sa +classmates.com +compassionandchoices.org +atlassian.com +merck.com +ispgateway.de +giallozafferano.it +wapka.me +idref.fr +shat.net +poste.it +rmmcdn.com +thefoxnation.com +getspeedbrowserp.com +retentionscience.com +batstrading.com +bradescoimoveis.com.br +dimestore.com +adklo.com +postlets.com +softsonic.net +softonicads.com +mobileoversee.net +howtogeek.com +bestbuy-jobs.com +xosothantai.com +tin.it +ocdn.eu +partsearch.com +aarki.net +berniaga.com +shareholder.com +sendeyim.com +app.lk +tvtropes.org +edgar-online.com +aams.it +123pay.vn +desmotivaciones.es +cepro.com +mediasetitalia.it +sltrib.com +asos-media.com +mirtesen.ru +banzai.it +infoplease.com +csnphilly.com +artisantools.com +workopolis.com +jazzedcdn.com +subiz.com +blogabull.com +internetvideoarchive.com +tf1.fr +websteroidsapp.com +boomrat.com +syncstatsdata.com +inrixmedia.com +bwbx.io +oboom.com +aliceposta.it +minecraftforums.net +dcbfjs.info +torrentdownloads.net +dogangazetecilik.com.tr +odometer.com +ringtonematcher.com +mmnetwork.mobi +line.me +creativeapis.com +kadinvekadin.net +crwd.io +activerain.com +nationalenquirer.com +pnas.org +menshealth.com +virusfree.cz +theartoflivingbetter.com +klimg.com +9nl.cc +examinerontopic.com +puretracks.com +ticketweb.com +imdbws.com +imlive.com +nwave.de +videotender.com +cio.com +businesscatalyst.com +lumosity.com +mongodb.org +noisey.com +ccomrcdn.com +socdm.com +best2tol.com +marriott-email.com +blammoservers.com +ssa.gov +telmex.net +cargurus.com +statig.com +qqmail.com +ultimateclassicrock.com +businessinsider.my +skynet.be +ccgslb.com.cn +lan.com +full.sc +kioskea.net +thevoterguide.org +locationlabs.com +gambleaware.co.uk +chiltepin.net +facebookmail.com +westga.edu +twitthis.com +differencegames.com +thesmokinggun.com +adoftheyear.com +appnext.com +fashiontmes.com +signupgenius.com +bluemediappc.com +whitepagesinc.com +turninc.com +hcuge.ch +futureplc.com +ukashal.com.tr +adtpix.com +counter-strike.net +geni.us +micromaxonline.com +bright.net +toshiba.com +bitshare.com +bigcharts.com +cnet.co.kr +leadzu.com +forbesmiddleeast.com +a-static.com +ink361.com +xobni.com +changeip.com +ndnmediaservices.com +cwtv.com +ticketsnow.com +kongcdn.com +jumptaps.com +iadvize.com +instagr.am +flirchicdn.com +thdws.com +torontosun.com +plansmedihealthsolutions.me +openvpn.net +strawpoll.me +pubdirecte.com +givalike.org +leechers-paradise.org +openoffice.org +pleer.com +caixaseguros.com.br +whitepagescustomers.com +dowjonesonline.com +tivi988.com +amctv.com +heartinternet.uk +homeshop18.com +da3e3.net +celebrityselfy.com +securepageloader.com +ourtime.org +bidtrk.com +dlink.com.tw +123-reg.co.uk +intelliad.com +bbcmundo.com +marktplaats.nl +fineartamerica.com +lininteractive.com +gocricket.com +brainfall.com +sblk.io +bandcamp.com +marfeel.com +joy.ac +artlebedev.ru +house.gov +rai.it +theguardian.tv +aliqin.cn +saurik.com +pushpin.com +fimserve.com +mezzobit.com +hulkshare.com +viewalytics.com +beeimg.com +ad-serverparc.nl +anpdm.com +hip2save.com +myegy.com +doisotrung.com +medyanet.net +giltcdn.com +xxxbunker.com +e5.sk +metavertising.com +posta.com.tr +infosbelges.eu +atlanticbb.net +medianewsgroup.com +bimbolive.com +realgravity.com +leaseweb.net +dhgate.com +xosominhngoc.com +yahoomail.com +frgimages.com +alipayobjects.com +cdnbd.com +ibibo.com +uvnimg.com +appleinsider.com +banerator.net +prserv.net +veniso.com +assineglobo.com.br +taggify.net +dailyveso.com +ajillionmax.com +wrightsmedia.com +adups.cn +gogoanime.com +bradescopoderpublico.com.br +skybet.com +thetvdb.com +mystartantiphishing.com +neimanmarcus.com +northcountrypublicradio.org +adupmediaxml.com +fastcdn.com +startappservice.com +archives.gov +edgedatg.com +unica.com +kakao.co.kr +xfinitytv.com +piksel.com +pornfeedback.com +fowar.net +afip.gov.ar +acsalaska.net +cackle.me +download-ap.com +istockimg.com +slashfilm.com +tmdb.org +intelius.com +cnevids.com +omgfacts.com +comicbook.com +rai.tv +justin.tv +arin.net +edreams.it +sedo.com +ifilez.org +lookcpm.com +carhartt.com +cogeco.ca +bitreactor.to +musicradar.com +ulogin.ru +demandforce.com +fluidsurveys.com +casasbahia.com.br +expediajobs.com +amerikanki.com +videoweed.es +weeklyfinancialsolutions.com +janrain.ws +ttmikro.com +runnersworld.com +emdep.vn +celebdirtylaundry.com +codeproject.com +bathandbodyworks.com +seriouseats.com +handmark.com +miitbeian.gov.cn +funplusgame.com +freepp.com +tubecup.org +spanishcentral.com +oregon.gov +fearthesword.com +newslook.com +pressdisplay.com +zara.com +corpmailsvcs.com +afcdn.com +canoe.ca +databyacxiom.com +clickpoint.com +thepioneerwoman.com +aviationweather.gov +hiphopmyway.com +popularmechanics.com +vertica.com +kliksaya.com +bbcgoodfood.com +ccc.se +gfi.com +everydayfamily.com +jiathis.com +ndtvimg.com +wowslider.com +worlderror.org +parcelstream.com +cifraclubnews.com.br +iafrica.com +thinglink.me +darkbluev2.com +sradserver.com +hotmart.net.br +altova.com +financialcontent.com +despegar.com +bradesconikkei.com.br +successfactors.com +blackplanet.com +fox.com.tr +icoco.com +fastserv.com +thehartford.com +mangahere.co +ticketmaster.ie +groovorio.com +detroitbadboys.com +saavn.com +linio.com +passionfruitads.com +apache.org +trendinglifestyles.com +novamov.com +appcelerator.net +emlfiles4.com +takataka.vn +dreamspark.com +cricbuzz.com +ixl.com +plosone.org +k-12techdecisions.com +yhoo.it +hgtvremodels.com +google.com.bn +backblaze.com +stubhubstatic.com +geektyrant.com +vinsight.de +ilmessaggero.it +timewarnercable.com +emltrk.com +miui.com +clickhole.com +natura.com.br +thetimes.co.uk +foxtvmedia.com +listhub.net +jinx.com +appmessages.com +uproxxcdn.com +bby.com +zargan.com +ble.ac +cquotient.com +megaupload.com +allhiphop.com +sify.com +mcssl.com +rewardstyle.com +zohostatic.com +cedexis-test.com +homeadvisor.com +megamailservers.com +termtutor.com +forbes.co.il +tasteofhome.com +telegraaf.nl +searspartsdirect.com +secunia.com +right-ads.com +readability.com +shop.pe +argos.co.uk +empireonline.com +vesochieuxo.com +adpassback.com +oneallcdn.com +kienthuc.net.vn +alternet.org +app-adforce.jp +tracelytics.com +meetmecdna.com +renfe.com +targetingmantra.com +wwbads.com +xyxpk.com +coolmath-games.com +adki.com +cyclingnews.com +minhacasamelhor.com.br +ahram.org.eg +almesryoon.com +gamefaqs.net +arabseed.com +cineplex.com +dha.com.tr +quoracdn.net +geewa.com +cdnhost2000xl.com +newbayasp.net +dayup.org +fshare.vn +icd9data.com +paytm.com +veoh.com +adforgames.com +postimage.org +play.it +huff.to +antevenio.com +tuaw.com +digitalcameraworld.com +t3.com +muscache.com +gamecloudnetwork.com +alfadevs.com +apptornado.com +csrlbs.com +vfpress.vn +link.net +kmpmedia.net +synchronychat.com +tencentmind.com +utexas.edu +msgapp.com +hostip.info +fidelityinvestments.com +hearthhead.com +screenrant.com +keeng.vn +cricketcb.com +google.tt +thinkgeek.com +arginfo.com +radzolo.com +hinkhoj.com +elcomercio.pe +get.com +ehow.de +ftc.gov +websiteprotegido.com.br +lider.cl +mitula.net +gu.com +upqzfile.com +bigleaguestew.com +lolnexus.com +cam4support.com +broadwayworld.com +collegesportslive.com +gmfleet.com +ivillage.com +probioslim.com +etherealhakai.com +gbtv.com +getkeepsafe.com +vast.com +a10.com +bbcamerica.com +bj.com.br +tipeez.com +zwaar.org +g.co +broadvid.com +stuff.co.nz +adapd.com +tinviet360.com +mademan.com +traktum.com +sparknotes.com +amazinglytimedphotos.com +rk.com +qip.ru +lhssfj.com +indiewire.com +sodahead.com +liversely.net +granify.com +touchtype-fluency.com +pgol.it +linkbucksmedia.com +mangareader.net +kimg.cn +pepsico.com +tecmarketing.com +trafex.net +milevo.com.br +cantv.net +active.com +oneandone.net +terraempresas.com.br +smugmug.com +yeah1.com +tvrage.com +aionfreetoplay.com +northgrum.com +globalpost.com +memecenter.com +sunbelt-software.com +searsoutlet.com +bbcomcdn.com +r24-tech.com +el-balad.com +cnbcmediasales.com +playtika.com +healthoks.cf +software-cdn.net +torrentz.ch +wapkaimage.com +tdcanadatrust.com +cnsnews.com +alumniconnections.com +eanalyzer.de +panoramtech.net +bikeradar.com +adfeedstrk.com +healthbk.ga +egencia.com +h2porn.com +webhostsy.com +newshost.co.za +porntube.com +quadranet.com +discogs.com +seslisozluk.net +skyteam.com +nenipxex.org +tiydhrpes.info +ayosdito.ph +yourlust.com +michigan.gov +gimokxo.org +nzznw.info +quill.com +topgear.com +epattu.net +proofpoint.com +adrd.co +espressonline.it +brazzersnetwork.com +wordcentral.com +clustrmaps.com +reimanpub.com +appshopper.com +tmbtzyha.net +pplive.com +ryanair.com +360buy.com +usc.edu +naver.net +zenaps.com +wmyjwfixhk.net +rising.cn +sporxtv.com +responsetap.com +nutrend.com +agrantsem.com +aoicyowsk.org +faqhk.net +greenbot.com +cam4bucks.com +seagateshare.com +gmasxewuon.com +dnsmadeeasy.com +vvrqnaibg.com +vywadxft.org +whowhatwear.com +ykxdinmt.com +ddnebnwogv.info +hgrjubwklk.info +dpgcquf.info +azqitla.org +wvjxq.info +smqtpt.com +qsstats.com +ctygsgsfgus.org +kaktjc.org +tzfbqzbmq.net +ucjyuasw.org +bxcma.info +ztat.net +qsgymo0vb6.com +theknot.com +costco-static.com +kmsethnz.com +pearsoncmg.com +ovlqhgc.org +ruamrckswcm.com +urekamedia.com +opensecrets.org +gametrailers.com +xnwjp.info +kotane.info +sievdlstgmh.org +adrbtvpzot.org +fkfazhyyzy.org +libraryh3lp.com +nyfzelqtlwz.info +terena.org +christianmingle.com +apmebf.com +zewvyymdbud.net +hdakqysubl.com +bigfishgames.com +bananarepublic.com +yuzzsfhd.net +bn.tl +mcmlytjw.com +skkkzsym.net +thgqo.info +xkrlmshkbhi.net +fduqkswpbx.org +comodoca4.com +stargazete.com +xcwxufd.org +tbppdpkd.com +fuspdqhon.com +ophxwklr.com +fbi.gov +nejm.org +jungroup.com +gbawcd.net +hjaqpjm.net +iryrmxjtpoy.org +sickbeard.com +hcubxxbg.net +rkhvtlc.info +sxqlfkjxyoa.com +syhttygog.com +dsmmadvantage.com +suntrust.com +eosumfx.org +wjrusyiws.net +pimsleurapproach.com +zxqjbnqbl.org +ojarpdtabs.org +yavli.com +wimwbh.net +audioadcenter.com +adplus.co.id +pqnbkt.org +pacgliym.org +smaclick.com +suuynm.org +tbbujqkry.net +tjzfkph.org +beatport.com +infinite-scroll.com +specialsituationsurvey.com +wiroos.com +amapmksw.net +hschn.net +nawkxuj.net +sfknlqcy.net +coremobility.com +quinnipiac.edu +zwhjex.net +vanillaforums.com +dworusea.org +hgexer.org +slimg.com +data-url.com +kndmdemzoyo.org +kxwloaxw.com +veybrms.com +learnersdictionary.com +guonmkwd.org +vusgcs.org +qqmpvwycwu.org +thaeragkyt.com +barstoolsports.com +stjude.org +nzduxi.cc +trpxl.com +rmbmbebtpp.com +ticketmaster.ca +idg.com +rcqgdkxzpwg.org +lucianne.com +emediate.ch +gesylwir.net +gmfdyonl.cc +idgbszogmfl.info +ijwozlv.info +myaffiliates.com +royalbank.com +slashgear.com +portalmore.com +dgzgpy.net +mifnpd.com +atkrwcld.com +fieuctzm.info +hocmdvia.com +hoopz.co.in +daqiqpbi.org +kjrh.com +redbus.in +puxue.com +txcdn.cn +uuzyaid.org +xjgeqznqm.cc +xvhvc.net +lottomatica.it +mkt51.net +dipkxlq.cc +recipechart.com +uicdn.com +wbodgchdwfh.net +ziffdavisinternational.com +viedij.cc +dmi.gov.tr +dobjzh.org +khmsfmbw.net +ltsbmuenq.net +abc7news.com +adprotected.com +iglqp.org +marieclaire.it +clrxzewdc.com +ewtxsmpeh.com +forbesmagazine.es +tqetpiijm.net +fcsinsider.com +bbokjmcdle.cc +jboujpnbbm.net +echiui.com +kyivpost.com +rtbid.me +fcontrol.com.br +abdbs.org +edaily.vn +tiebaimg.com +nvhrlq.cc +bcbits.com +uc123.com +forbes.ro +freeonlinegames.com +beyjhow.cc +iblbc.cc +played.to +smartbrief.com +webutation.net +metaps.com +zbwdqg.net +pzpdrxc.org +neweggimages.com +mwmkkvpozzk.net +reclameaqui.com.br +lircxaievm.com +whdwsmbkyob.cc +aoltech.com +smarterfox.com +someecards.com +momagic.mobi +piwhzmuw.cc +bdllppws.com +bubiocgy.cc +epmjp.cc +quintelligence.com +finra.org +roost.me +ixijwtzr.cc +web-ster.com +netsuite.com +hurra.com +wcsrtmfao.com +lonelyplanet.com +rbcroyalbank.com +qulgueozpfy.cc +etoro.com +vartoken.com +hetwnddmrii.cc +metrocast.net +sportsauthority.com +valeculturacaixa.com.br +ojnmbxtt.cc +pcgamesn.com +yvvodqsqlu.com +boingboing.net +newscred.com +nepzzveo.cc +zonelabs.com +vnpgroup.net +cganz.org +jahvpwqx.cc +businessinsider.co.id +armorgames.com +nqofpsk.cc +topify.com +dafiti.com.br +gatech.edu +instantssl.com +zorjxg.cc +xmladfeed.com +telerikstatic.com +cofktwvu.cc +zohomail.com +adstrckr.net +christianpost.com +albjkdomro.info +xerox.com +holder.com.ua +hdnxn.info +qbwbzmtv.cc +shefinds.com +editorandpublisher.com +ahqkb.cc +chhsekhqw.com +unidadeditorial.es +gamefree.la +centos.org +addfreestats.com +navy.mil +systemaffiliate.com +dgyhxlxpji.cc +elfagr.org +qeugpfgmgj.cc +wjejmd.cc +connectedly.com +demandforced3.com +searshomeservices.com +admsjycuykv.cc +nszose.biz +ttlbd.net +vvmducso.cc +indir.com +sun-sentinel.com +alfynetwork.com +burgerbusiness.com +complexmediainc.com +adpay.com +solutionzip.info +yroytop.cc +kshwtj.com +kldvhndinht.cc +ttzoroahi.biz +vietbao.vn +securenetconnection.com +magazineluiza.com.br +osuosl.org +fzofzn.cc +iiisgr.info +rarlab.com +urge.com +toroadvertisingmedia.com +wgeprggwv.ws +realclearreligion.org +webhst.com +actiontec.com +tbzdnwk.cc +wegotthiscovered.com +ad4push.com +ilpost.it +realclearpolicy.com +racinggamer.com +khvjh.cc +rtuvrdso.com +a2pub.com +tanidigital.com +eoccqzwk.info +ppbgu.cc +rogysyzp.org +adprudence.com +hudl.com +snknfcfp.info +wkpevwftzv.com +ooma.com +uetcfrdm.info +pdfforge.org +realcleartechnology.com +kwlgtm.cc +ojrfqwt.cc +htzwnl.biz +oaokwaah.info +reklaam.co +vvkux.org +imgix.net +dscww.net +anvato.com +daphnecm.com +wqaxikwy.cc +whatismyip.com +pro-football-reference.com +bitsnoop.com +hldiw.org +hsotptmpg.info +kbebwxy.org +pymrinle.net +i.ua +rwkczaox.info +1digitalstock.com +oxcfwwerbxd.com +rjtgbmjei.info +ljybyhejei.ws +dumpaday.com +socialnewsdesk.com +kdexbh.biz +nxlwx.com +turbobytes.com +smzln.info +livemixtapes.com +lqokojzey.info +deepdyve.com +lfvhjb.biz +pziio.biz +vejeizob.info +hbqtq.net +ijunutaf.net +rvnkxm.org +astrology.com +fiserv.com +finam.ru +onlineservice2013.org +foodnetworkstore.com +vietcombank.com.vn +xsbkh.org +dgvfpdudj.ws +hepfmg.info +qbxurzdb.org +realclearhistory.com +zunrobjo.cc +slack.com +umhbxlgc.net +amo.vn +fullizle.org +ngads.com +acs86.com +baalwpbn.biz +campaigner.com +uprieyivrgd.info +blzojuvragg.biz +edptolrs.net +eetsp.ws +gdmvctuqky.biz +shopifyapps.com +qowatfdl.info +ahasvfxzc.net +worthlossfatseasily.me +evanguard.com +ghconduit.com +hmvsndleo.info +iypflxli.net +jmkdxyk.ws +adbooth.net +vdyarquq.info +spin.com +hruhac.org +izrsbbrtdqn.info +lrylelyr.com +viralgains.com +rsrobt.org +wqutrzfd.biz +wtfdyo.org +adpublik.com +gyini.org +huebwztdp.com +kxieadw.org +ilsole24ore.com +awe.sm +bcwvlnkx.org +cgrlveisaam.net +zebestof.com +egxfslsii.org +nevytf.ws +nsjce.org +draftkings.com +xwygma.biz +edtoroziecr.org +esndmtix.org +kgnhx.org +unbouncepages.com +dorkly.com +xwmfbz.org +irduxdivnc.ws +homedepotfoundation.org +xvylary.ws +ajyoux.org +hissage.com +jzcchvae.org +lxikgoxptag.org +petapixel.com +maketutorial.com +craftsman.com +blldnoxvi.ws +dwdkim.org +jueux.ws +qbcotemhrcj.biz +rfgzqaji.biz +csbew.com +cynnacwuo.info +ltxzb.net +webserviceline.org +gnvkexqvt.ws +lkgtos.biz +moe.gov.eg +realcleardefense.com +serviceonlinetech.org +expediafranchise.com +ilgiornale.it +thesuperficial.com +cogentco.com +tuklyreb.org +presselite.com +crioojfv.net +fxtubwo.com +gyfezbzowuw.biz +qixxrais.ws +cherylstyle.com +ajbjz.org +btmglunma.ws +epnnazri.org +fwfsqcux.net +ifidfszesh.net +jipillw.ws +jetbrains.com +zuncwgq.ws +broadagesports.com +romnz.biz +yovkuwwd.net +dpjksji.info +fqxyheiqp.org +qmfcjsyjpvg.ws +tzzqzosshyj.ws +uiezsksf.ws +vhbvo.org +theaustralian.com.au +zaman.com.tr +xdpenvsi.net +yzifhqrk.ws +cihjafxcp.net +dqyrtyya.ws +mlgzkzwwnz.org +rvgqpud.com +vimdsspys.net +wgupyqdndw.ws +gcmforex.com +wzgzpehhnkm.ws +xbhygm.org +xoqovau.com +dkoshhap.net +rdk.al +crdrjs.info +tsjfn.biz +ebates.com +aitarget.ru +grxhhiqszb.biz +hwokelamsqp.net +sfdoqpsw.info +talkingdata.net +bitebbs.com +academia-assets.com +allocine.net +24o.it +theresumator.com +ncfqy.org +escapistmagazine.com +paginebianche.it +cardstar.mobi +fdovetlp.ws +hfhkqlsevi.biz +ieppg.net +qwghivalvbb.ws +lasa.com.br +rrxzsi.ws +jykchlbyvr.ws +qbktqkl.ws +tiqdfh.biz +billionairesaustralia.com +zpfdtwgyfq.biz +nortel.com +djibbxypely.org +nl-img.com +mbqxxiyr.info +moijzbt.com +ydstatic.com +visa.com +ecgdjumtk.biz +autotraderstatic.com +gmjstqdpmv.org +hbdezkxzjf.ws +sgphffta.cc +timeforkids.com +fnjbgcdmlfv.com +rqrrzwj.net +nyc.gov +socialsecurity.gov +bdawooytpv.org +imgbox.com +optaim.com +judgepedia.org +adtrixi.com +unsmcp.ws +webtraxs.com +hilton.com +ddhvwhqg.biz +hellogiggles.com +huhujreo.cc +hzjga.biz +mhzprtm.biz +tkwff.ws +frontpagemag.com +alphamaletribe.com +hheyqpnuchm.biz +rtk.io +rederecord.com.br +jfkazzj.ws +mniku.com +rwqmopqgak.biz +xnaocbyr.com +ads-ex.com +mobilefuse.net +g2trk.com +advg.jp +oxhks.cc +cmmdkgthw.info +ypimblaegg.ws +betradar.com +glancecdn.net +codeandtheory.com +estrelando.com.br +cctalk.vn +igrmpr.biz +dmmotion.com +app.com +zstwjoeptfu.biz +imagetwist.com +iuvkikajb.net +torrentbox.com +ycmewgipmtn.cc +owlewvrivgz.cc +bmbcmh.org +mhyjyrgn.biz +newclientgenservice.com +wvniza.org +arfuxfliw.net +cxnynydz.biz +xhpcyboz.ws +accweopv.info +egsjzjpz.cc +sociedadedenegocios.com.br +skqtdpgseun.com +epalaxghv.cc +qhirhxxowcf.cc +vericlk.com +eroeooof.com +fkyehkcmxx.com +guuwouduwgk.com +netaffiliation.com +h-cdn.co +nvvknqpt.com +gruppoespresso.it +blmqccfb.biz +phvtxypi.cc +clickable.net +jlqnseshyfr.com +idvaultservices.com +moving.com.br +rehcuqjlszg.ws +wikipedia.com +arcfdtls.net +topfreegames.com +qufacnib.cc +eouvh.cc +getanxhkfl.org +sf.net +nextinsure.com +joinecsc.com +createsend1.com +jcloud.com +hboqdalzdb.cc +iucfpstqju.net +racked.com +haichuanmei.com +pofvc.cc +aeriagames.com +popoholic.com +zxeraykcru.biz +ataiswtjq.cc +imaginecup.com +wxelvbutl.cc +erzubfwdpid.cc +madnet.ru +aqlrq.cc +metrics34.com +fitnesskeeperapi.com +osmsvcxwgh.cc +orlandosentinel.com +tf2outpost.com +xuvkaipwcdb.cc +rackspace.com +publitalia.it +aicsuc.cc +publishthis.com +vrzuthlz.cc +bbcurdu.com +plala.or.jp +afqtrggqe.ws +allakhazam.com +ccmbenchmark.com +foxitsoftware.com +cktfpxeyhq.cc +lubebgyh.cc +minfils.eu +instantcheckmate.com +vmofcpi.cc +ufggdwezyd.cc +youtube.it +dzxgristcfg.ws +hjskmeltj.cc +saraiva.com.br +conmio.com +duomi.com +marcamarca.com.tr +dpupdate.com +alesouza.com +canstockphoto.com +static-nextag.com +mycapture.com +stupiddope.com +xosnetwork.com +jetveopbmzo.cc +peixeurbano.com.br +www.nhs.uk +politicususa.com +timeincnewsgroupcustompub.com +yenisafak.com.tr +home.com +appshat.com +exoticads.com +ift.tt +livejournal.net +lyrta.cc +trfirmaekle.com +cloud9-media.net +pogo.com +gkhroqza.cc +homefinder.com +just-downloads.net +gpqwrwmgist.cc +discuz.net +fractalsciences.com +ovkmjiw.cc +flingguru.com +tds.net +tdqdghjtnj.cc +blogtamsu.vn +weather.com.cn +ckstatic.com +josscdn.com +onion.com +gquldikg.ws +vqwcgak.ws +adcdnx.com +cannedbanners.com +symnds.com +ucla.edu +gilt.com +nme.com +beachfrontio.com +laughingsquid.com +wpxi.com +marktest.pt +lzpwgq.biz +castfire.com +rtb-media.ru +techbargains.com +qugylddujwj.biz +zagat.com +bdiaydynor.biz +mslearn.net +fiesdacaixa.com.br +jhdiknjlq.cc +soyouthinkyoucangame.com +rightnowtech.com +vzagof.ws +axcogulnxj.ws +addgsene.cc +pornmarathon.com +identityguard.com +juksr.com +statuspage.io +hiapk.com +ugcroceao.biz +fastpic.ru +anninhthudo.vn +batpmturner.com +newtentionassets.net +bouncebidder.com +ktqhyn.biz +vjyfw.biz +interactivedata.com +yyzhroqelh.ws +massrel.io +nmgx.co.uk +statistik-gallup.net +topsy.com +ksl.com +automattic.com +dhqfg.ws +fkxzw.biz +mypoints.com +91.com +kjmtpknc.ws +infoescola.com +infg.com.br +wral.com +tianmidian.com +mgaserv.com +sgshbsnxw.biz +xvhibyfku.cc +mightynova.com +1anh.com +rentalcars.com +infonet.vn +hsbfgc.biz +columbia.edu +internetsegura.org +nwfeybp.ws +nzuxayxvb.biz +jjijdzz.biz +mboeughth.biz +athleta.com +ml.com +elfvt.biz +porch.com +ncaa.com +cultofmac.com +searsholdings.com +chztfneh.biz +jimwqv.ws +xrtmbe.biz +cutun.vn +bango.net +feodpyusmel.ws +safetynutbe.com +kcna.co.jp +realclearbooks.com +sendspace.com +datatables.net +containerstore.com +frontdb.com +ybiqqrrr.biz +binaryprofessional.com +easysol.net +cornell.edu +truex.com +scrippsnetworksdigital.com +xzuai.biz +nckrnpudwgc.ws +rbcdn.com +iilcuaks.ws +cafemomstatic.com +care2.com +keepcalm-o-matic.co.uk +dictionaryapi.com +edgussbrehp.ws +mangafox.me +nwoxjixrm.biz +shiftyjelly.com.au +topix.com +btypevb.ws +dcstkgbi.cn +developmaster.in +ulmjklxf.biz +webserviceline2013.org +fvgjmz.biz +motortrend.com +eehnrwsg.biz +truehits.in.th +persona.org +cybergolf.com +nastyvideotube.com +adual.net +krrhvyjsbiq.biz +kenmore.com +nordstromrack.com +pfgvgnvk.ws +spoonful.com +techsonlineervice.org +rockpapershotgun.com +365dm.com +ivwbox.de +tiki.vn +wabagmti.ws +destinydb.com +cafeland.vn +realclearenergy.org +com2us.net +onlineservicetest.org +xtgem.com +jkovqerv.biz +oxqgnhu.ws +babyzone.com +unjklmh.biz +advancedigital.com +gaana.com +homedepotemail.com +pmlatam.com +qsrqqa.ws +msgf.net +mudah.my +reviversoft.com +twincities.com +uolcontent.com +adedgemedia.com +e-karaman.com +army.mil +elgwpdbdz.biz +ktyqltpace.biz +vwislcpb.ws +baseball-reference.com +2xbpub.com +cuteo.vn +playappstats.com +wgt.com +cipebk.biz +dhresource.com +thingsremembered.com +fgnbrfxt.ws +netu.tv +nitropdf.com +gsecondscreen.com +segsrvcs.com +graytvinc.com +brasilpost.com.br +imvu.com +xalo.vn +zromhyh.biz +plaync.co.kr +neulion.net +topkit.com +ghzyehci.ws +71.am +pkqyv.ws +lexisnexis.com +awpjtkvmhd.cn +mtnldelhi.in +omgpm.com +oppomobile.vn +tripadvisor.com.br +dealnews.com +iqhgbjuzi.biz +osatcxntrug.ws +nwsource.com +selectablemedia.com +toucharcade.com +achdebit.com +deseretnews.com +pussycash.com +robbreport.com +zcloud.io +nmfsuibwt.cn +pdfalmta.ws +fmtrader.com +techtimes.com +101affiliates.com +wonderwall.com +xmlclick-g.com +dumlfnhd.ws +ihotdjn.cn +medu.com +eqnextfans.com +telcel.com +rzwvyhv.biz +videonhadat.com.vn +wsi.com +oxforddictionaries.com +commonshare.net +bilyoner.com +wxljto.cn +crbfjs.info +gccdn.net +ccsend.com +ajansspor.com +adsboxonline.com +bqfhmcnsolt.ws +newdemoonlinecloud.com +wnxiwg.cn +reviewjournal.com +sportsnet.ca +borsahaber.com +hfzic.cn +nwacmz.cn +britannicaenglish.com +progressive.com +vogue.com +ocn.ad.jp +mqqmaavwqul.cn +toshiba-tro.de +didyn.co.uk +adonly.com +sp.gov.br +regiedepub.com +india.gov.in +umd.edu +ebscohost.com +aslangamestudio.com +fypmh.cn +cobalt.com +lowes.ca +rvzrjs.info +rxmkklx.cn +bahldhghl.cn +hyuvwsj.cn +abear.com.br +realcleareducation.com +cgfmfa.cn +macysinc.com +expedient.net +hzrxlnynak.cn +zihrsowdavb.cn +pwshqtanxpi.cn +bzwhwzur.biz +bradescoabrasuaconta.com.br +gptxzy.cn +petstocking.com +disneybaby.com +tubexclips.com +electnext.com +scexbsrw.cn +xnsports.com +ey.com +yielm.com +gssdnyiq.info +ygrskyqd.cn +animetoon.tv +lastampa.it +yourlustmedia.com +hhhfwrv.cn +efnet.org +charterbusiness.com +businessinsider.sg +districtwest.com +qljqtnmqx.cn +say.ac +rafflecopter.com +bradescocelular.com.br +goodsearch.com +siemens.com +socialquantum.ru +green-label.com +adwhirl.com +instructure.com +twcc.com +adtop.vn +muachung168.com +freewebs.com +personalcreations.com +pgoamedia.com +marketo.com +ndl.go.jp +clickdesk.com +playdom.com +paramountcommunication.com +sm3na.com +depotliive.in +adbucks.com +directtoustore.com +encontreobb.com.br +xpxbmzqcpma.cn +iaveqvyuo.cn +squid-cache.org +bsqptibskvk.cn +gluftlsdqtc.cn +vidbull.com +ecvjixc.cn +vginyzu.cn +fid-inv.com +letvimg.com +dominionenterprises.com +hotplug.ru +androidauthority.com +safarishop.com.br +fastestcdn.net +easytaxi.com.br +pornwhite.com +fitsugar.com +electric.net +yoomeegames.com +cjsyvlh.cn +violetgrey.com +trivago.com +viacom.com +walmartonline.com.ar +adtdp.com +eastmoney.com +bzmqb.cn +fannation.com +baodautu.vn +ukzrhfbn.cn +vietid.net +mygofer.com +gamersmedia.com +mailcontrol.com +getdownloadmy.com +ninpblt.cn +justfab.com +casasbahia-imagens.com.br +loopassets.net +asocials.com +myntassets.com +tweetriver.com +aerserv.com +bsecure.com +in-appadvertising.com +clientstatsservice.com +ticketfly.com +azdjforhire.com +jqueryui.com +radio-canada.ca +elo7.com.br +planet49.com +iac.com +dolphin-browser.cn +singnet.com.sg +sugarops.com +cartoonnetwork.com +pages01.net +ssl.com +vgoeun.cn +cisive.net +namehub.com +torrents.to +charitynavigator.org +yallakora.com +poletracker.org +cddbp.net +rapgenius.com +speedanalysis.net +eltiempo.es +yatra.com +pgpartner.com +mochitot.com +sendtonews.com +u-on.eu +hallmark.com +cloudmagic.com +contextly.com +bonzaii.no +vchat.vn +jimstatic.com +houstonchronicle.com +seiyu.co.jp +lifeselector.com +streamtip.com +jihadwatch.org +whirlpoolcorp.com +kraftfoods.com +bhphoto.com +56.com +woolik.com +indo.net.id +rcn.net +korabia.com +chicagobusiness.com +powerjobs.com +cbeyond.com +buscapecompany.com +sarenza.com +pornleech.me +kudzu.com +dirmusiic.in +datingfactory.net +disneyjunior.com +torhead.com +greatdepothomey.asia +wmo.int +elle.it +tirerack.com +aftonbladet-cdn.se +iwebar.com +rabilitan.com +4tube.com +prevention.com +kelleybluebookimages.com +tdbank.com +xskt.com.vn +democlientnet.com +toolserver.org +vads.vn +fuse.tv +picdn.net +softwareprojects.com +xbmc.org +thenewrepublic.com +starwars.com +allperfectlytimed.com +education.com +altmetric.com +watchseries.ag +ultimate-guitar.com +ads.cc +worldoftanks.com +zzz.vn +dhs.gov +mc.gov.br +abload.de +nudevector.com +jungledisk.com +aetn.com +the-best-adults-vine.com +quixapp.com +ciudad.com.ar +written.com +cox.com +paragaranti.com +cityfeet.com +vanityfair.it +piperlime.com +rncdn3.com +pjtra.com +browsemark.net +torrenti.al +odcdn.com +acessoainformacao.gov.br +apnic.net +k7computing.com +vplay8.com +d3head.com +arvixe.com +thumbshots.com +kronos.com +gpstream.net +traviangames.com +cursos24horas.com.br +msftconnecttest.com +olapic.com +dhcxjscg.cn +adhaven.com +justgetflux.com +clicktracks.com +adserverpub.com +zqjjrpx.com +rgoskspdu.cn +the-m-age.com +nsf.gov +despegar.com.ar +esohead.com +publicpolicypolling.com +stitcher.com +mdotm.co +sage.com +internic.com +btrd.net +cafebiz.vn +herezera.com +srds.com +uefa.com +newsbank.com +barracudanetworks.com +librato.com +backstage.com +veesible.it +abebooks.de +baifendian.com +wsjsafehouse.com +alistmoz.cn +adfootprints.com +oyunmoyun.com +redbookmag.com +userneeds.dk +bowl.com +vs.com +tinvn.info +imdb.de +arsmtp.com +ipapp.com +tracki112.com +freerepublic.com +starpulse.com +marieclaire.com +yjthmjbjie.cn +meowapi.com +surfingbird.ru +dailytech.com +startimes.com +tgadvapps.it +nero.com +cloudy.ec +fund123.cn +enuygun.com +vervemobile.com +vesselapp.com +finans.dk +ppliowlh.cn +ealojs.cn +trustedshops.com +audioware.com.br +newsnow.co.uk +salaoautocaixa.com.br +deseretconnect.com +vozforums.com +wwpcitfsg.cn +example.com +budgettravel.com +google.hu +ukcompfindlove.info +imagefap.com +bellsouth.com +keystealth.org +adduplex.com +vclnrnhfn.cn +adjug.com +vzaar.com +mmoui.com +decider.com +transpera.com +cprpt.com +zincx.com +directallapp.in +rampanel.com +scmplayer.net +axhldab.cn +bbcpersian.com +gymplan.com +todoist.com +glhxefai.cn +lfdsddbga.cn +ystdcru.cn +loop11.com +clevernet.vn +nos.nl +redhat.com +mediamatters.org +zoneedit.com +rjqmczlucxd.cn +anycash.com +flwarmwg.cn +sbc.com +unitusaforalllove.info +photoscape.org +eemqepu.cn +monsternotebook.com.tr +yyqtbvqv.cn +cegjobs.com +xbnfrg.cn +xplosion.de +taylorswift.com +newspaperdirect.com +gznwldaxh.cn +stardoll.com +tcpdiag.net +fitpregnancy.com +xmypgoqokb.cn +bellmedia.ca +getbills.com +incredibar.com +lowesforpros.com +pdpdvtec.cn +scielo.br +ovoadv.com +iahzw.cn +akqstjbu.cn +lqw.me +multiview.com +wjfrewfykf.cn +marksandspencer.com +trackingclick.net +bangbros.com +mymailwall.com +tntvffmm.cn +topbongda.com +vervewireless.com +animenewsnetwork.com +zvab.com +zgzfbgq.cn +assineabril.com +plaync.com +qmvxa.cn +jumia.com.eg +bsd.net +juzkqm.cn +ksgks.cn +snapworkapps.com +dfna.net +tradenet.net +elsevier.com +predictormedia.com +insidercarnews.com +thisiscolossal.com +dreamhost.com +sexsearch.com +wajuvzaq.cn +muthead.com +pulsepoint.com +movielink.com +talktalk.net +gigaset.net +dijimecmua.com +fit-predictor.net +dlgokzzejj.cn +loopme.me +markitondemand.com +libreoffice.org +omp.me +rankingsandreviews.com +politicopro.com +thegioididong.com +mobizone.mobi +ctpost.com +samsungmediahub.net +sdphruvn.cn +i-funbox.com +saveur.com +jdjfdsnd.cn +clanacion.com.ar +egywcfyz.cn +cityspark.com +snapapp.com +intpvbjj.cn +appyet.com +coolmath.com +zoosk.com +escapemg.com +rtbfy.com +dropboxatwork.com +dwtyj.cn +adidas.com +gogames.me +bt.com +vng.vn +buzzle.com +liquida.it +theage.com.au +grandparents.com +startpage24.com +network18online.com +phird.cn +adobecc.com +openstat.ru +freshdesk.com +delivery53.com +aljazeera.net +zvelo.com +voxel.net +idtargeting.com +alwafd.org +twenga.it +aa.com.tr +btmbxiacvl.cn +registrar-servers.com +bradescorural.com.br +govtrack.us +cb2.com +chel.su +torrentsnipe.info +hitsk.in +ehealthcaresolutions.com +wowinterface.com +schwabcdn.com +frenchmid.eu +newsbytes.com +betterbythemin.com +bigfishsites.com +booksamillion.com +spigjs.info +etymonline.com +supertelafilmesonlinegratis.com +yottaa.net +websimages.com +minhavida.com.br +dietaesaude.com.br +kameleoon.com +livepromotools.com +perfectcitytime.com +hunts.com +molliemakes.com +bloggercomment.com +brookings.edu +bm324.com +sitewit.com +intsig.net +mirror-image.net +foodonthetable.com +ticketmaster.com.au +kernel.org +vancouversun.com +adtegrity.net +brightcloud.com +swappa.com +politiken.dk +hollywoodtuna.com +popfixx.com +fareportal.com +followhorseracing.com +immunet.com +sterling-adventures.co.uk +mgyun.com +baofeng.com +mastercms.org +magiq.com +projone.net +anchorfree.net +usafis.org +salaodocarro.com.br +consumerinput.com +linkonlineworld.com +cc.com +futurity.org +easy-ads.com +adelixir.com +arenafootball.com +mawaly.com +cloudantivirus.com +albawabhnews.com +istoedinheiro.com.br +self.com +pushauction.com +qnsr.com +enchantedlearning.com +gigenet.com +crimtan.com +skem1.com +leaseweb.com +noobmeter.com +idexx.com +theaccept.net +extra.com.br +bimedia.net +thepennyhoarder.com +trackeame.com +gophoto.it +vt.edu +doveclub.it +fptshop.com.vn +gamebaby.net +pangora.com +friv-games.com +locaweb.com.br +polarnavy.com +ado-global.com +incmd07.com +sancohuyenthoai.vn +ulketv.com.tr +esa.int +rescuetime.com +trustedform.com +hotlog.ru +privacystar.com +trademob.com +ssrn.com +kidshealth.org +dostor.org +hotair.com +kidsfootlocker.com +ebay-mediacentre.co.uk +mapmyfitness.com +adform.com +society6.com +hotdeal.vn +thevideo.me +duolingo.com +gossipcenter.com +checkpointsys.com +swacargo.com +cpcache.com +goobzo.com +gtburst.com +cameraprive.com.br +2dopeboyz.com +admixclicks.com +aggeliopolis.gr +fa8072.com +lowescreativeideas.com +playerio.com +findthebest.com +tira.cn +plentyoffish.com +ximad.com +csnne.com +golden-goose-method.com +opm.gov +quattroruote.it +faqs.org +snssdk.com +vdict.com +wpimg.pl +doubletwist.com +inbox.com +cdn-redfin.com +bluecoat.com +kayak.co.uk +cwmods.com +timesonline.co.uk +direcpc.com +redtailtechnology.com +wayreview.com +h12-media.net +meterserver.vn +cisp.com +dulichhue.com.vn +sannhac.com +buenosearch.com +technetevents.com +michaels.com +thesimplethings.com +scoutanalytics.net +laptopmag.com +ticketmaster.es +aerisapi.com +groupon.co.uk +hotukdeals.com +ofuxico.com.br +vidyomani.com +yahoo.co.uk +ledsmagazine.com +blogsonyxperia.com.br +docusign.net +good.com +blogcatalog.com +dhl.com +tagesschau.de +hulatoo.net +sonypictures.com +masralarabia.com +spigtrdpjs.info +berkeley.edu +corrieredellosport.it +guitarbattle.com.br +smartorrent.com +dattobackup.com +cfcloudcdn.com +rte.ie +netvigator.com +genius.com +triangleoffense.com +rifthead.com +cobaltnitra.com +ganadineroconencuestas.com +panorama.it +beaconads.com +hostingxtreme.com +sohu.com.cn +hanmail.net +rhapsody.com +esoui.com +picasion.com +parade.com +socialgrowthtechnologies.com +saigonamthuc.vn +beliefnet.com +pressurenet.io +valaffiliates.com +klart.se +supereva.com +adhitzads.com +audioaddict.com +toparcadehits.com +maquinadevendas.com.br +staticontent.com +gpo.gov +sipc.org +ticketmaster.com.mx +85dcf732d593.se +podoweb.net +wzrkt.com +forumfree.net +wikispaces.com +lapresse.ca +bluehornet.com +eq2interface.com +lostandfound.aero +ttdt.vn +comicbookmovie.com +onlinebackupsolution.com +m-viet.com +cnnic.cn +lowes.com.mx +otmsrv.com +nosc.us +mayo.edu +go.im +rainbowtgx.com +fastrapid.in +locamail.com.br +tachthongtin.com +spongecdn.com +activejunky.com +celebrityhd.tv +lifehack.org +callcentric.com +ibs.it +adglue.com +imagehost123.com +jagranjosh.com +checkm8.com +voyeurhit.com +addictinggames.com +digitalfuture.com +beedoctor.vn +washtimes.com +entwine-wines.com +eqinterface.com +uglab.org +csnwashington.com +produzindoeventos.com.br +accoona.com +policypedia.org +triradar.com +nintendo.com +developermedia.com +easportsfifaworld.com +letv.cn +laposte.net +weather.ca +trafficserving.com +quickconnect.to +fedoraproject.org +thegrio.com +ziraat.com.tr +hitwebcounter.com +select-n-go.com +bigtorrent.org +bradescopromotora.com.br +adverline.com +dota2lounge.com +trade101.com +kitconet.com +bidsystem.com +khampha.vn +rvchsr.com +deployads.com +capital.it +cliktrue.com +bitsontherun.com +legalmail.it +ultradns.com +comsenz.com +iolo.net +goodnet.org +laweekly.com +stereogum.com +leadboltads.net +babycenter.ca +tctmobile.com +fundsspeedy.in +bbccanada.com +flic.kr +icbc.com.cn +photoshop.com +recipezaar.com +dayzdb.com +livrariasaraiva.com.br +clickdiagnostic.com +mygame82.com +demandbase.com +mailhop.org +townsquareblogs.com +smarsh.com +tinnong.vn +vagas.com +thottbot.com +cargocollective.com +app47.mobi +freelotto.com +justuno.com +motiwecdn.com +blogtoplist.com +payplay.fm +morgdm.ru +publiabril.com.br +sidecubes.com +gamezone.com +monitus.net +linkd.in +fastenal.com +cookappsgames.com +yenikadin.com +bcove.me +fancy.com +ad4mat.net +ilmeteo.com +estrongs.com +hamburgdeclaration.org +autoexpress.co.uk +disneycareers.com +sumotracker.org +newclientstaticsrv.com +mail2world.com +ed.gov +kayak.co.in +youwatch.org +aggregateknowledge.com +140proof.com +teamsnap.com +nend.net +avazutracking.net +w.org +sethads.info +computershopper.com +sweetcaptcha.com +alcatel-lucent.com +kayak.com.br +digitalthrottle.com +southwestthemagazine.com +futuredial.com +linksrs.com +kayak.de +textnow.com +uzmanreklam.com +sendgrid.com +unicode.org +vg247.com +installmac.com +stocktonport.com +voxer.com +asksemtools.com +cdn-image.com +pressherald.com +bpsecure.com +fastapi.net +ricardoeletro.com.br +ustatik.com +cat.com +bongda24h.vn +alohaenterprise.com +forumfree.it +capitaliq.com +lotrointerface.com +wothic.com +advancedhosters.com +tsunami.gov +webgozar.ir +gboxapp.com +bradescouniversitarios.com.br +beyeu.com +gothamist.com +crispadvertising.com +motiveadserver.com +hi-pi.com +bradescoseguranca.com.br +canlitv.com +macysjobs.com +vidigital.ru +globomarcas.com.br +curalate.com +aroofquote.info +financialpost.com +hightail.com +ldscdn.org +millry.co +tmocache.com +iomartmail.com +webtv.net +kayak.com.au +shipmentmanager.com +wetpaint.com +imf.org +zite.com +browsehappy.com +cambridge.org +kidsafeseal.com +basbakanlik.gov.tr +n11.com.tr +mibet.com +thefrugalgirls.com +classicvacations.com +lifeatexpedia.com +images4us.com +kayak.ch +mayoclinic.com +netcommunities.com +torchbrowserjs.info +shawcable.net +wired.it +carbonhousehost2.com +kayak.it +sigalert.com +msedge.net +enigmaadserver.com +harryanddavid.com +shopzilla.com +p0y.cn +lilluna.com +cdn-seekingalpha.com +kayak.com.ar +riftui.com +catalinahub.com +eonli.ne +freakshare.com +carrentals.com +easycounter.com +harrenmediatools.com +salesmore.pl +openfeint.com +sitespeeds.com +vitalk.vn +whydoiseetheads.info +dvdvideosoft.com +duke.edu +bubblestat.com +active-srv02.de +movshare.net +goapk.com +gazeteoku.com +sleazyneasy.com +ecbsn.com +pontofrio.com.br +fling.com +huffson.com +umich.edu +dmcimg.com +ntvsp.org +immobiliare.it +muscleandfitness.com +aetndigital.com +shopop.me +kayak.com.hk +usajobs.gov +szgpbgnmexpx6.com +aksam.com.tr +yadi.sk +utah.edu +asktiava.com +olx.co.id +mozdev.org +mail.com +swtorui.com +pub1.us +vinhomes.vn +gazzabet.it +shopandroid.com +sourceforge.jp +spoti.fi +kayak.es +costcophotocenter.com +electronichouse.com +newgrounds.com +smarttech.com +landsofamerica.com +adinfo-guardian.co.uk +247msg.com +assoc-amazon.co.uk +photoshelter.com +hersheys.com +gryphonet.com +geewa.net +catve.tv +lemde.fr +vstarcam.com +thezoereport.com +channel4.com +stroeerdigitalmedia.de +pornleech.ru +radikal.ru +benjerry.com +homedesigntreasure.com +katestube.com +hiconversion.com +dotaoutpost.com +kayak.com.mx +maximumpc.com +modcloth.com +800hosting.com +eyeblaster.com +live365.com +datamind.ru +fvap.gov +yieidmanager.com +kayak.dk +listhub.com +tns-cs.net +kayak.fr +udemy.com +bancodoplaneta.com.br +lpcdn.ca +gourmetads.com +aastocks.com +architecturaldigest.com +mnetads.com +barracudacentral.com +comingsoon.net +kayak.com.tr +vizio.com +leonardoadv.it +freeskreen.com +inn.ru +trckng.net +pixstatic.com +staplesrewardscenter.com +ezanga.com +fastcolabs.com +teklinks.com +iprimus.com.au +c4tw.net +cms.gov +host-engine.com +umtrack.com +zacks.com +di.sn +ietf.org +camdolls.com +oyungemisi.com +disneylandparis.com +appgratuites-network.com +townsquaremedia.com +mediative.com +commentarymagazine.com +crazycashformula.net +grupaonet.pl +playnomics.net +icann.org +bikeqwikfix.com +mobtada.com +vrbo.com +silkroad.com +123c.vn +vietad.vn +edline.net +yesadsrv.com +getfirebug.com +markandgraham.com +newegg.ca +swafreedomshop.com +com.com +formesuabanda.com.br +magisto.com +mapbar.com +brimg.net +canlibahissiteleri24.com +synxis.com +adyoulike.com +costco.ca +pressly.com +doorsteps.com +clkbid.com +cyveillance.com +musicnet.com +mrnumber.com +arenabg.com \ No newline at end of file diff --git a/qa/scripts/perf/able/values/political_parties.txt b/qa/scripts/perf/able/values/political_parties.txt new file mode 100644 index 000000000..d8a9434b8 --- /dev/null +++ b/qa/scripts/perf/able/values/political_parties.txt @@ -0,0 +1,7 @@ +Democrat +Republican +Independent +Libertarian +Green +Federalist +Whig \ No newline at end of file diff --git a/qa/scripts/runSamsungGauntlet.sh b/qa/scripts/runSamsungGauntlet.sh index 37cc0a930..6c1822c91 100644 --- a/qa/scripts/runSamsungGauntlet.sh +++ b/qa/scripts/runSamsungGauntlet.sh @@ -2,9 +2,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - # requires TF_VAR_cluster_prefix env var to be set if [ -z ${TF_VAR_cluster_prefix+x} ]; then echo "setting TF_VAR_cluster_prefix"; diff --git a/qa/scripts/runSmokeTest.sh b/qa/scripts/runSmokeTest.sh index 6b598995e..8d4ff1923 100755 --- a/qa/scripts/runSmokeTest.sh +++ b/qa/scripts/runSmokeTest.sh @@ -2,10 +2,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - # requires TF_VAR_cluster_prefix env var to be set if [ -z ${TF_VAR_cluster_prefix+x} ]; then echo "setting TF_VAR_cluster_prefix"; diff --git a/qa/scripts/setupSamsungGauntlet.sh b/qa/scripts/setupSamsungGauntlet.sh index e7929ea9f..1e6060f2e 100755 --- a/qa/scripts/setupSamsungGauntlet.sh +++ b/qa/scripts/setupSamsungGauntlet.sh @@ -3,9 +3,6 @@ # To run script: ./setupSamsungGauntlet.sh export TF_IN_AUTOMATION=1 -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/utilCluster.sh diff --git a/qa/scripts/setupSmokeTest.sh b/qa/scripts/setupSmokeTest.sh index 1059e5f29..997974494 100755 --- a/qa/scripts/setupSmokeTest.sh +++ b/qa/scripts/setupSmokeTest.sh @@ -3,9 +3,6 @@ # To run script: ./setupSmokeTest.sh export TF_IN_AUTOMATION=1 -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/utilCluster.sh diff --git a/qa/scripts/teardownSmokeTest.sh b/qa/scripts/teardownSmokeTest.sh index 76eeb564b..21e9f390a 100755 --- a/qa/scripts/teardownSmokeTest.sh +++ b/qa/scripts/teardownSmokeTest.sh @@ -2,9 +2,6 @@ # To run script: ./teardownSmokeTest.sh -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - cd qa/tf/ci/smoketest export TF_IN_AUTOMATION=1 terraform destroy -auto-approve diff --git a/qa/scripts/testSmokeTest.sh b/qa/scripts/testSmokeTest.sh index b6ba34e7f..b27b12931 100755 --- a/qa/scripts/testSmokeTest.sh +++ b/qa/scripts/testSmokeTest.sh @@ -1,8 +1,5 @@ #!/bin/bash -# requires TF_VAR_branch env var to be set -if [ -z ${TF_VAR_branch+x} ]; then echo "TF_VAR_branch is unset"; else echo "TF_VAR_branch is set to '$TF_VAR_branch'"; fi - SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source $SCRIPT_DIR/utilCluster.sh diff --git a/qa/tf/.modules/featurebase-cluster/main.tf b/qa/tf/.modules/featurebase-cluster/main.tf index d21fbd4df..c06576025 100644 --- a/qa/tf/.modules/featurebase-cluster/main.tf +++ b/qa/tf/.modules/featurebase-cluster/main.tf @@ -13,7 +13,7 @@ data "aws_ami" "amazon_linux_2" { filter { name = "architecture" - values = ["arm64"] + values = var.fb_cluster_arch } } @@ -38,6 +38,7 @@ resource "aws_instance" "fb_cluster_nodes" { volume_type = var.fb_data_disk_type volume_size = var.fb_data_disk_size_gb iops = var.fb_data_disk_iops + encrypted = true } tags = { @@ -70,6 +71,7 @@ resource "aws_instance" "fb_ingest" { volume_type = var.fb_ingest_disk_type volume_size = var.fb_ingest_disk_size_gb iops = var.fb_ingest_disk_iops + encrypted = true } tags = { @@ -255,6 +257,27 @@ resource "aws_iam_role" "fb_cluster_node_role" { }) } + inline_policy { + name = "s3_perms" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "VisualEditor0", + Effect = "Allow", + Action = ["s3:PutObject", "s3:GetObject"], + Resource = "arn:aws:s3:::molecula-perf-storage/*" + }, + { + Sid = "VisualEditor1", + Effect = "Allow", + Action = "s3:PutObject", + Resource = "arn:aws:s3:::molecula-artifact-storage/*" + } + ] + }) + } + tags = { Prefix = "${var.cluster_prefix}" Name = "${var.cluster_prefix}-fb_cluster_node_role" diff --git a/qa/tf/.modules/featurebase-cluster/variables.tf b/qa/tf/.modules/featurebase-cluster/variables.tf index 4d7762d7e..7f7bf2e95 100644 --- a/qa/tf/.modules/featurebase-cluster/variables.tf +++ b/qa/tf/.modules/featurebase-cluster/variables.tf @@ -3,6 +3,11 @@ variable "cluster_prefix" { description = "This is a identifier that will be prefixed to created resources" } +variable "fb_cluster_arch" { + type = list(string) + default = ["arm64"] +} + variable "fb_ingest_type" { type = string default = "c6g.2xlarge" diff --git a/qa/tf/ci/smoketest/terraform.tfstate.backup b/qa/tf/ci/smoketest/terraform.tfstate.backup deleted file mode 100644 index bc199358f..000000000 --- a/qa/tf/ci/smoketest/terraform.tfstate.backup +++ /dev/null @@ -1,710 +0,0 @@ -{ - "version": 4, - "terraform_version": "1.1.2", - "serial": 220, - "lineage": "0f5e8a05-0e94-e86f-f384-26086bd40585", - "outputs": { - "cluster_prefix": { - "value": "gauntlet-wFQOOzXB51R3ebr", - "type": "string" - }, - "data_node_ips": { - "value": [ - "10.0.1.16" - ], - "type": [ - "tuple", - [ - "string" - ] - ] - }, - "fb_cluster_replica_count": { - "value": 1, - "type": "number" - }, - "ingest_ips": { - "value": [ - "3.142.172.86" - ], - "type": [ - "tuple", - [ - "string" - ] - ] - } - }, - "resources": [ - { - "module": "module.ci-cluster", - "mode": "data", - "type": "aws_ami", - "name": "amazon_linux_2", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "architecture": "arm64", - "arn": "arn:aws:ec2:us-east-2::image/ami-088e1f338c3b87d1a", - "block_device_mappings": [ - { - "device_name": "/dev/xvda", - "ebs": { - "delete_on_termination": "true", - "encrypted": "false", - "iops": "0", - "snapshot_id": "snap-0f9ae89577e61b172", - "throughput": "0", - "volume_size": "8", - "volume_type": "gp2" - }, - "no_device": "", - "virtual_name": "" - } - ], - "creation_date": "2022-01-05T21:55:03.000Z", - "description": "Amazon Linux 2 LTS Arm64 AMI 2.0.20211223.0 arm64 HVM gp2", - "ena_support": true, - "executable_users": null, - "filter": [ - { - "name": "architecture", - "values": [ - "arm64" - ] - }, - { - "name": "name", - "values": [ - "amzn2-ami-hvm-*" - ] - }, - { - "name": "virtualization-type", - "values": [ - "hvm" - ] - } - ], - "hypervisor": "xen", - "id": "ami-088e1f338c3b87d1a", - "image_id": "ami-088e1f338c3b87d1a", - "image_location": "amazon/amzn2-ami-hvm-2.0.20211223.0-arm64-gp2", - "image_owner_alias": "amazon", - "image_type": "machine", - "kernel_id": null, - "most_recent": true, - "name": "amzn2-ami-hvm-2.0.20211223.0-arm64-gp2", - "name_regex": null, - "owner_id": "137112412989", - "owners": [ - "amazon" - ], - "platform": null, - "platform_details": "Linux/UNIX", - "product_codes": [], - "public": true, - "ramdisk_id": null, - "root_device_name": "/dev/xvda", - "root_device_type": "ebs", - "root_snapshot_id": "snap-0f9ae89577e61b172", - "sriov_net_support": "simple", - "state": "available", - "state_reason": { - "code": "UNSET", - "message": "UNSET" - }, - "tags": {}, - "usage_operation": "RunInstances", - "virtualization_type": "hvm" - }, - "sensitive_attributes": [] - } - ] - }, - { - "module": "module.ci-cluster", - "mode": "managed", - "type": "aws_iam_instance_profile", - "name": "fb_cluster_node_profile", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::977373308795:instance-profile/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "create_date": "2022-01-12T15:00:15Z", - "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "name_prefix": null, - "path": "/", - "role": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "tags": null, - "tags_all": {}, - "unique_id": "AIPA6HD75E55WG6AVJLA4" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "module.ci-cluster.aws_iam_role.fb_cluster_node_role" - ] - } - ] - }, - { - "module": "module.ci-cluster", - "mode": "managed", - "type": "aws_iam_role", - "name": "fb_cluster_node_role", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::977373308795:role/gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "assume_role_policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}", - "create_date": "2022-01-12T15:00:12Z", - "description": "", - "force_detach_policies": false, - "id": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "inline_policy": [ - { - "name": "ec2_read_all", - "policy": "{\"Statement\":[{\"Action\":[\"ec2:Describe*\"],\"Effect\":\"Allow\",\"Resource\":\"*\"}],\"Version\":\"2012-10-17\"}" - } - ], - "managed_policy_arns": [], - "max_session_duration": 3600, - "name": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node", - "name_prefix": "", - "path": "/", - "permissions_boundary": null, - "tags": null, - "tags_all": {}, - "unique_id": "AROA6HD75E55YDDESCKUN" - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - }, - { - "module": "module.ci-cluster", - "mode": "managed", - "type": "aws_instance", - "name": "fb_cluster_nodes", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "index_key": 0, - "schema_version": 1, - "attributes": { - "ami": "ami-088e1f338c3b87d1a", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-03a8897456f88b2a4", - "associate_public_ip_address": false, - "availability_zone": "us-east-2a", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 2, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-0028ea6c90c8ed849", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-03a8897456f88b2a4", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.large", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-0deaddc1bee29d648", - "private_dns": "ip-10-0-1-16.us-east-2.compute.internal", - "private_ip": "10.0.1.16", - "public_dns": "", - "public_ip": "", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": null, - "throughput": 125, - "volume_id": "vol-00eb233b46ec8746f", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-050b1219d78f2db1b", - "tags": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "cluster_node" - }, - "tags_all": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-cluster-0", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "cluster_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": null, - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-060f8471c4271acb8" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.ci-cluster.aws_iam_role.fb_cluster_node_role", - "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.ci-cluster.aws_security_group.featurebase", - "module.ci-cluster.data.aws_ami.amazon_linux_2" - ] - } - ] - }, - { - "module": "module.ci-cluster", - "mode": "managed", - "type": "aws_instance", - "name": "fb_ingest", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "index_key": 0, - "schema_version": 1, - "attributes": { - "ami": "ami-088e1f338c3b87d1a", - "arn": "arn:aws:ec2:us-east-2:977373308795:instance/i-012b6f6ad4a0295a8", - "associate_public_ip_address": true, - "availability_zone": "us-east-2a", - "capacity_reservation_specification": [ - { - "capacity_reservation_preference": "open", - "capacity_reservation_target": [] - } - ], - "cpu_core_count": 2, - "cpu_threads_per_core": 1, - "credit_specification": [], - "disable_api_termination": false, - "ebs_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/sdb", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "snapshot_id": "", - "tags": {}, - "throughput": 125, - "volume_id": "vol-0278897f78cb4f99e", - "volume_size": 100, - "volume_type": "gp3" - } - ], - "ebs_optimized": false, - "enclave_options": [ - { - "enabled": false - } - ], - "ephemeral_block_device": [], - "get_password_data": false, - "hibernation": false, - "host_id": null, - "iam_instance_profile": "gauntlet-wFQOOzXB51R3ebr-fb_cluster_node_profile", - "id": "i-012b6f6ad4a0295a8", - "instance_initiated_shutdown_behavior": "stop", - "instance_state": "running", - "instance_type": "m6g.large", - "ipv6_address_count": 0, - "ipv6_addresses": [], - "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "launch_template": [], - "metadata_options": [ - { - "http_endpoint": "enabled", - "http_put_response_hop_limit": 1, - "http_tokens": "optional" - } - ], - "monitoring": true, - "network_interface": [], - "outpost_arn": "", - "password_data": "", - "placement_group": "", - "placement_partition_number": null, - "primary_network_interface_id": "eni-0bb5e02c328f3c371", - "private_dns": "ip-10-0-101-66.us-east-2.compute.internal", - "private_ip": "10.0.101.66", - "public_dns": "", - "public_ip": "3.142.172.86", - "root_block_device": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "encrypted": false, - "iops": 3000, - "kms_key_id": "", - "tags": null, - "throughput": 125, - "volume_id": "vol-073cdccfd75958e27", - "volume_size": 20, - "volume_type": "gp3" - } - ], - "secondary_private_ips": [], - "security_groups": [], - "source_dest_check": true, - "subnet_id": "subnet-066b4b922b54e51a2", - "tags": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "ingest_node" - }, - "tags_all": { - "Name": "gauntlet-wFQOOzXB51R3ebr-featurebase-ingest-0", - "Prefix": "gauntlet-wFQOOzXB51R3ebr", - "Role": "ingest_node" - }, - "tenancy": "default", - "timeouts": null, - "user_data": null, - "user_data_base64": null, - "volume_tags": null, - "vpc_security_group_ids": [ - "sg-07e67c395f920042c" - ] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6MTIwMDAwMDAwMDAwMCwidXBkYXRlIjo2MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjEifQ==", - "dependencies": [ - "module.ci-cluster.aws_iam_instance_profile.fb_cluster_node_profile", - "module.ci-cluster.aws_iam_role.fb_cluster_node_role", - "module.ci-cluster.aws_key_pair.gitlab-featurebase-ci", - "module.ci-cluster.aws_security_group.ingest", - "module.ci-cluster.data.aws_ami.amazon_linux_2" - ] - } - ] - }, - { - "module": "module.ci-cluster", - "mode": "managed", - "type": "aws_key_pair", - "name": "gitlab-featurebase-ci", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:key-pair/gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "fingerprint": "69:e5:4b:15:8d:1f:22:61:de:08:a9:ee:f3:29:5c:69", - "id": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "key_name": "gauntlet-wFQOOzXB51R3ebr-gitlab-ci", - "key_name_prefix": "", - "key_pair_id": "key-0a49a5ef950bc7f0c", - "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC91hhpVHNonAG7ku2ugpxEskf9KHeyHJPQJT26OHrMUw7R+T5A8TjqSzTau07sXQ/E9SO3ebV8SJ5PqeaQOnQB8VEvVNK0DjQH7ppvNg1Rfs42FZT9ttzTMvOjsSbK3vZTHXdoKQEdC9NxBwSkFIRGQojK1HUOq9xGrw31fA1OjSwlpLcbx7yyg18lcqW6UOptnVR8U9Yy9qQ5jZF1HtkQ6L9J+gv4o1UyNAUK2bopeGiXpBc3PQ/CFaFT2h/aqLBP66qAHsHVyAFD3PIRtplC5EHa8jXDgLacEls0uF7Q3kRPxvzcuo4g4VkOn1rDy9qH3vd2hT3aKVnM73FIDUiL", - "tags": null, - "tags_all": {} - }, - "sensitive_attributes": [], - "private": "eyJzY2hlbWFfdmVyc2lvbiI6IjEifQ==" - } - ] - }, - { - "module": "module.ci-cluster", - "mode": "managed", - "type": "aws_security_group", - "name": "featurebase", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-060f8471c4271acb8", - "description": "Allow featurebase inbound traffic", - "egress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "", - "from_port": 0, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "-1", - "security_groups": [], - "self": false, - "to_port": 0 - } - ], - "id": "sg-060f8471c4271acb8", - "ingress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "SSH", - "from_port": 22, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 22 - }, - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "icmp from Anywhere", - "from_port": -1, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "icmp", - "security_groups": [], - "self": false, - "to_port": -1 - }, - { - "cidr_blocks": [ - "10.0.0.0/16" - ], - "description": "etcd from internal 2", - "from_port": 10401, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10401 - }, - { - "cidr_blocks": [ - "10.0.0.0/16" - ], - "description": "etcd from internal", - "from_port": 10301, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10301 - }, - { - "cidr_blocks": [ - "10.0.0.0/8", - "172.31.0.0/16" - ], - "description": "GRPC from Internal", - "from_port": 20101, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 20101 - }, - { - "cidr_blocks": [ - "10.0.0.0/8", - "172.31.0.0/16" - ], - "description": "HTTP from Internal", - "from_port": 10101, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10101 - }, - { - "cidr_blocks": [ - "10.0.0.0/8", - "172.31.0.0/16" - ], - "description": "PostgreSQL from Internal", - "from_port": 55432, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 55432 - } - ], - "name": "gauntlet-wFQOOzXB51R3ebr-allow_featurebase", - "name_prefix": "", - "owner_id": "977373308795", - "revoke_rules_on_delete": false, - "tags": { - "Name": "allow_featurebase" - }, - "tags_all": { - "Name": "allow_featurebase" - }, - "timeouts": null, - "vpc_id": "vpc-05a26a122f961dc2b" - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - }, - { - "module": "module.ci-cluster", - "mode": "managed", - "type": "aws_security_group", - "name": "ingest", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:ec2:us-east-2:977373308795:security-group/sg-07e67c395f920042c", - "description": "Allow ingest inbound traffic", - "egress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "", - "from_port": 0, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "-1", - "security_groups": [], - "self": false, - "to_port": 0 - } - ], - "id": "sg-07e67c395f920042c", - "ingress": [ - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "", - "from_port": 10101, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 10101 - }, - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "SSH", - "from_port": 22, - "ipv6_cidr_blocks": [ - "::/0" - ], - "prefix_list_ids": [], - "protocol": "tcp", - "security_groups": [], - "self": false, - "to_port": 22 - }, - { - "cidr_blocks": [ - "0.0.0.0/0" - ], - "description": "icmp from Anywhere", - "from_port": -1, - "ipv6_cidr_blocks": [], - "prefix_list_ids": [], - "protocol": "icmp", - "security_groups": [], - "self": false, - "to_port": -1 - } - ], - "name": "gauntlet-wFQOOzXB51R3ebr-allow_ingest", - "name_prefix": "", - "owner_id": "977373308795", - "revoke_rules_on_delete": false, - "tags": { - "Name": "allow_ingest" - }, - "tags_all": { - "Name": "allow_ingest" - }, - "timeouts": null, - "vpc_id": "vpc-05a26a122f961dc2b" - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - } - ] -} diff --git a/qa/tf/ci/smoketest/variables.tf b/qa/tf/ci/smoketest/variables.tf index 578305341..a327ea4ff 100644 --- a/qa/tf/ci/smoketest/variables.tf +++ b/qa/tf/ci/smoketest/variables.tf @@ -13,7 +13,3 @@ variable "cluster_prefix" { description = "This is a identifier that will be prefixed to created resources" } -variable "branch" { - type = string - description = "The branch we are on" -} \ No newline at end of file diff --git a/qa/tf/gauntlet/samsung/variables.tf b/qa/tf/gauntlet/samsung/variables.tf index 578305341..e55c7936d 100644 --- a/qa/tf/gauntlet/samsung/variables.tf +++ b/qa/tf/gauntlet/samsung/variables.tf @@ -12,8 +12,3 @@ variable "cluster_prefix" { type = string description = "This is a identifier that will be prefixed to created resources" } - -variable "branch" { - type = string - description = "The branch we are on" -} \ No newline at end of file diff --git a/qa/tf/perf/able/main.tf b/qa/tf/perf/able/main.tf new file mode 100644 index 000000000..514f52173 --- /dev/null +++ b/qa/tf/perf/able/main.tf @@ -0,0 +1,17 @@ +module "able-cluster" { + source = "../../.modules/featurebase-cluster" + cluster_prefix = var.cluster_prefix + region = var.region + profile = var.profile + fb_data_node_type = "m6g.12xlarge" + fb_data_disk_iops = 10000 + fb_data_node_count = 3 + fb_ingest_type = "m6g.2xlarge" + fb_ingest_disk_iops = 10000 + fb_ingest_disk_size_gb = 500 + fb_ingest_node_count = 1 + vpc_id = "vpc-05a26a122f961dc2b" + vpc_cidr_block = "10.0.0.0/16" + vpc_public_subnets = ["subnet-066b4b922b54e51a2","subnet-037b8884269a69025","subnet-08482631514426210",] + vpc_private_subnets = ["subnet-0319dde319380326f","subnet-0517ca9a646d80f88","subnet-05a7b685ed27eb1cf",] +} \ No newline at end of file diff --git a/qa/tf/perf/able/outputs.tf b/qa/tf/perf/able/outputs.tf new file mode 100644 index 000000000..43acb92ab --- /dev/null +++ b/qa/tf/perf/able/outputs.tf @@ -0,0 +1,19 @@ +output "ingest_ips" { + description = "List of ingest IPs" + value = module.able-cluster.ingest_ips +} + +output "data_node_ips" { + description = "List of data node IPs" + value = module.able-cluster.data_node_ips +} + +output "cluster_prefix" { + description = "The cluster prefix used" + value = module.able-cluster.cluster_prefix +} + +output "fb_cluster_replica_count" { + description = "The cluster replica count used" + value = module.able-cluster.fb_cluster_replica_count +} diff --git a/qa/tf/perf/able/provider.tf b/qa/tf/perf/able/provider.tf new file mode 100644 index 000000000..c0fc95d9d --- /dev/null +++ b/qa/tf/perf/able/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = var.region + profile = var.profile +} \ No newline at end of file diff --git a/qa/tf/perf/able/tf.auto.tfvars b/qa/tf/perf/able/tf.auto.tfvars new file mode 100644 index 000000000..ac6de62a6 --- /dev/null +++ b/qa/tf/perf/able/tf.auto.tfvars @@ -0,0 +1,2 @@ +region = "us-east-2" +profile = "service-terraform" \ No newline at end of file diff --git a/qa/tf/perf/able/variables.tf b/qa/tf/perf/able/variables.tf new file mode 100644 index 000000000..e55c7936d --- /dev/null +++ b/qa/tf/perf/able/variables.tf @@ -0,0 +1,14 @@ +variable "region" { + description = "The AWS region in which the VPC should be built" + type = string +} + +variable "profile" { + description = "The name of the AWS profile Terraform should use for auth." + type = string +} + +variable "cluster_prefix" { + type = string + description = "This is a identifier that will be prefixed to created resources" +} diff --git a/rbf.go b/rbf.go index b79b2eb03..5360958cb 100644 --- a/rbf.go +++ b/rbf.go @@ -223,6 +223,74 @@ func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (c // which is expensive in practice and only really useful occasionally. const sortedParanoia = false +type countResults struct { + changeCount int + err error +} + +// RemoveChannel provides a method of streaming in bits or positions and not requiring a large buffer like add and remove +// the bits are input via the posChanel and the results are returned via the retChannel +func (tx *RBFTx) RemoveChannel(index, field, view string, shard uint64, a chan uint64, resChan chan countResults) { + name := rbfName(index, field, view, shard) + var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. + var rc *roaring.Container + var hi uint64 + var lo uint16 + var err error + changeCount := 0 + i := 0 + for v := range a { + hi, lo = highbits(v), lowbits(v) + if hi != lastHi { + // either first time through, or changed to a different container. + // do we need put the last updated container now? + if i > 0 { + // not first time through, write what we got. + if rc == nil || (rc.N() == 0) { + err = tx.tx.RemoveContainer(name, lastHi) + if err != nil { + resChan <- countResults{0, errors.Wrap(err, "failed to remove container")} + return + } + } else { + err = tx.tx.PutContainer(name, lastHi, rc) + if err != nil { + resChan <- countResults{0, errors.Wrap(err, "failed to put container")} + return + } + } + } + // get the next container + rc, err = tx.tx.Container(name, hi) + if err != nil { + resChan <- countResults{0, errors.Wrap(err, "failed to retrieve container")} + return + } + } // else same container, keep adding bits to rct. + chng := false + rc, chng = rc.Remove(lo) + if chng { + changeCount++ + } + lastHi = hi + i++ + } + // write the last updates. + if rc == nil || rc.N() == 0 { + err = tx.tx.RemoveContainer(name, hi) + if err != nil { + resChan <- countResults{0, errors.Wrap(err, "failed to remove container")} + return + } + } else { + err = tx.tx.PutContainer(name, hi, rc) + if err != nil { + resChan <- countResults{0, errors.Wrap(err, "put to remove container")} + return + } + } + resChan <- countResults{changeCount, nil} +} func (tx *RBFTx) addOrRemove(index, field, view string, shard uint64, remove bool, a ...uint64) (changeCount int, err error) { if len(a) == 0 { return 0, nil diff --git a/rbf/cursor.go b/rbf/cursor.go index 41e9e4d4f..75b05b9bc 100644 --- a/rbf/cursor.go +++ b/rbf/cursor.go @@ -287,9 +287,6 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) { leafCell1 := ConvertToLeafArgs(cell.Key, cbm) // ConvertToLeafArgs returns leafCell1 with BitN and ElemN updated. - if err := c.tx.freePgno(pgno); err != nil { - return false, err - } return true, c.putLeafCell(leafCell1) } @@ -375,10 +372,18 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { newEstPageSize += in.Size() - len(readLeafCellBytesAtOffset(leafPage, readCellOffset(leafPage, elem.index))) } + // Use the fast path if we are not splitting pages and the container types are the same. + useFast := newEstPageSize+16 <= PageSize + if useFast && !isInsert { + if prev := readLeafCell(leafPage, elem.index); prev.Type != in.Type { + useFast = false + } + } + // Use an optimized routine to insert the leaf cell if we won't overflow. // We pad the estimate with 16 bytes because we do 8-byte alignment of // both the cell and the index. - if newEstPageSize+16 <= PageSize { + if useFast { return c.putLeafCellFast(in, isInsert) } @@ -402,6 +407,22 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { copy(cells[elem.index+1:], cells[elem.index:]) } else { + // FB-1239: Free bitmap page if replaced container is a bitmap pointer. + prev := cells[elem.index] + if prev.Type == ContainerTypeBitmapPtr { + if in.Type == ContainerTypeBitmapPtr { + if toPgno(in.Data) != toPgno(prev.Data) { // bptr-to-bptr with different bitmap pages + if err := c.tx.freePgno(toPgno(prev.Data)); err != nil { + return err + } + } + } else if in.Type != ContainerTypeBitmap { + if err := c.tx.freePgno(toPgno(prev.Data)); err != nil { + return err + } + } + } + if in.Type == ContainerTypeBitmap { cell = cells[elem.index] if cell.Type != ContainerTypeBitmapPtr { @@ -411,10 +432,10 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { } cell.Type = ContainerTypeBitmapPtr cell.Data = fromPgno(bitmapPgno) - // update the BitN too - cell.BitN = in.BitN cell.ElemN = in.ElemN } + // update the BitN regardless + cell.BitN = in.BitN } } @@ -433,6 +454,7 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { } cell.Data = fromPgno(bitmapPgno) } + cells[elem.index] = cell // Split into multiple pages if page size is exceeded. @@ -474,9 +496,11 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { writeCellN(buf[:], len(group)) offset := dataOffset(len(group)) + x := 0 for j, cell := range group { writeLeafCell(buf[:], j, offset, cell) offset += align8(cell.Size()) + x++ } if err := c.tx.writePage(buf[:]); err != nil { @@ -614,7 +638,6 @@ func (c *Cursor) deleteLeafCell(key uint64) (err error) { copy(cells[elem.index:], cells[elem.index+1:]) cells[len(cells)-1] = leafCell{} cells = cells[:len(cells)-1] - // Write cells to page. buf := allocPage() writePageNo(buf[:], elem.pgno) @@ -626,6 +649,7 @@ func (c *Cursor) deleteLeafCell(key uint64) (err error) { writeLeafCell(buf[:], j, offset, cell) offset += align8(cell.Size()) } + if err := c.tx.writePage(buf[:]); err != nil { return err } diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go index 9c5603a3a..2bb2f7d6f 100644 --- a/rbf/cursor_test.go +++ b/rbf/cursor_test.go @@ -2,6 +2,7 @@ package rbf_test import ( + "bytes" "io" "math/bits" "math/rand" @@ -556,6 +557,44 @@ func TestCursor_RLETesting(t *testing.T) { }) } +func TestCursor_BitmapBitN(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer tx.Rollback() + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + bmData := make([]uint64, 1024) + for i := range bmData { + bmData[i] = 0x5555555555555555 + } + ct := roaring.NewContainerBitmap(-1, bmData) + err := tx.PutContainer("x", 0, ct) + if err != nil { + t.Fatalf("error writing container: %v", err) + } + // Putting a container to the same slot, which is also a bitmap + // (rather than a BitmapPtr), while the existing cell is a BitmapPtr, + // may not update BitN correctly. + ct, _ = ct.Add(1) + err = tx.PutContainer("x", 0, ct) + if err != nil { + t.Fatalf("rewriting container: %v", err) + } + v, err := tx.Container("x", 0) + if err != nil { + t.Fatalf("getting container: %v", err) + } + c1 := v.N() + v.Repair() + c2 := v.N() + if c1 != c2 { + t.Fatalf("expected count %d, got %d", c2, c1) + } + tx.Commit() +} + func TestCursor_RLEConversion(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) @@ -852,8 +891,10 @@ func TestDumpDot(t *testing.T) { if err != nil { t.Fatal(err) } - rbf.Dumpdot(tx, 0, " ", os.Stdout) + var b bytes.Buffer + rbf.Dumpdot(tx, 0, " ", &b) } + func TestCursor_UpdateBranchCells(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) @@ -1206,3 +1247,63 @@ func TestForEachRange(t *testing.T) { t.Fatalf("expected empty container, but see %v values left: '%#v'", len(valmap), valmap) } } + +func TestCursor_PutContainer(t *testing.T) { + t.Run("BitmapToArray", func(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + tx := MustBegin(t, db, true) + defer tx.Rollback() + + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + + bmData := make([]uint64, 1024) + for i := range bmData { + bmData[i] = 0x5555555555555555 + } + if err := tx.PutContainer("x", 0, roaring.NewContainerBitmap(-1, bmData)); err != nil { + t.Fatal(err) + } + if err := tx.PutContainer("x", 0, roaring.NewContainerArray([]uint16{0})); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }) + + t.Run("BitmapToBitmap", func(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + tx := MustBegin(t, db, true) + defer tx.Rollback() + + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + + data0 := make([]uint64, 1024) + for i := range data0 { + data0[i] = 0x5555555555555555 + } + if err := tx.PutContainer("x", 0, roaring.NewContainerBitmap(-1, data0)); err != nil { + t.Fatal(err) + } + + data1 := make([]uint64, 1024) + for i := range data0 { + data1[i] = 0x7777777777777777 + } + if err := tx.PutContainer("x", 0, roaring.NewContainerBitmap(-1, data1)); err != nil { + t.Fatal(err) + } + + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }) +} diff --git a/rbf/db.go b/rbf/db.go index a6c248e5b..bd92f2271 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -261,7 +261,7 @@ func (db *DB) methodicalWALPageN(pageN int) (lastMeta int, err error) { } switch { case IsMetaPage(page): - lastMeta = i + lastMeta = i + 1 case IsBitmapHeader(page): // skip the bitmap page, which we can't usefully evaluate i++ @@ -411,17 +411,28 @@ func (db *DB) checkpoint() (err error) { // Close closes the database. func (db *DB) Close() (err error) { - // TODO(bbj): Add wait group to hang until last Tx is complete. + // mark db as closed, spawn a thing to wait for existing tx to drain, then + // release the lock so they CAN drain. We do this before getting the + // write lock, so if something else is waiting on rwmu.Lock, and will be + // competing with us, we can ensure that it'll exit out quickly. + db.mu.Lock() + db.opened = false + // wait for transactions to complete + ch := make(chan struct{}) + db.afterCurrentTx(func() { + close(ch) + }) + db.mu.Unlock() + <-ch // Wait for writer lock. db.rwmu.Lock() defer db.rwmu.Unlock() + // and main DB lock. db.mu.Lock() defer db.mu.Unlock() - db.opened = false - // Close mmap handle. if db.data != nil { if e := syswrap.Munmap(db.data); e != nil && err == nil { diff --git a/rbf/db_test.go b/rbf/db_test.go index 3e6bdd1b1..2b886677c 100644 --- a/rbf/db_test.go +++ b/rbf/db_test.go @@ -139,6 +139,90 @@ func TestDB_WAL(t *testing.T) { t.Fatal(err) } }) + + // initially this is just a cut and paste of the Halt test, except that + // we close the DB while the reads are still running. + t.Run("Close", func(t *testing.T) { + if testing.Short() { + t.Skip("-short enabled, skipping") + } + + config := rbfcfg.NewDefaultConfig() + config.MaxWALSize = 16 * rbf.PageSize + config.MaxWALCheckpointSize = 8 * rbf.PageSize + config.MinWALCheckpointSize = 4 * rbf.PageSize + + db := MustOpenDB(t, config) + + // Continuously run read overlapping transactions. + ctx, cancel := context.WithCancel(context.Background()) + g, ctx := errgroup.WithContext(ctx) + for i := 0; i < 10; i++ { + i := i + g.Go(func() error { + time.Sleep(time.Duration(i) * 10 * time.Millisecond) // stagger + for { + if err := ctx.Err(); err != nil { + return nil + } + + if err := func() error { + tx, err := db.Begin(false) + if err != nil { + return err + } + // give the db time to close between when we opened and + // when we run the Container call + time.Sleep(10 * time.Millisecond) + _, err = tx.Container("x", 0) + if err != nil { + t.Fatalf("requesting container: %v", err) + } + defer tx.Rollback() + return nil + }(); err != nil { + // it's okay to ErrClosed, because we plan to close + // the database out from under us. + if err != rbf.ErrClosed { + return err + } else { + return nil + } + } + } + }) + } + + // Generate updates to the DB/WAL. + for i := 0; i < 100; i++ { + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + + if err := tx.CreateBitmapIfNotExists("x"); err != nil { + t.Fatal(err) + } else if _, err := tx.Add("x", uint64(i)); err != nil { + t.Fatal(err) + } else if err := tx.Commit(); err != nil { + t.Fatal(err) + } + time.Sleep(1 * time.Millisecond) + }() + } + // close the db now. + err := db.Close() + if err != nil { + t.Fatalf("closing db: %v", err) + } + // delay a bit to let some readers try to read + time.Sleep(20 * time.Millisecond) + + // Stop read transactions & wait. + cancel() + if err := g.Wait(); err != nil { + t.Fatal(err) + } + }) } func TestDB_Recovery(t *testing.T) { diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go index 3ff80350b..0a2432128 100644 --- a/rbf/rbf_test.go +++ b/rbf/rbf_test.go @@ -164,79 +164,6 @@ func GenerateValues(rand *rand.Rand, n int) []uint64 { return a } -var _ = ToRows - -// ToRows returns a sorted list of rows from a set of values. -func ToRows(values []uint64) []*Row { - m := make(map[uint64][]uint64) - for _, v := range values { - id := v / rbf.ShardWidth - m[id] = append(m[id], v&rbf.RowValueMask) - } - - a := make([]*Row, 0, len(m)) - for id, values := range m { - a = append(a, &Row{ID: id, Values: values}) - } - sort.Slice(a, func(i, j int) bool { return a[i].ID < a[j].ID }) - return a -} - -var _ = Row{} - -type Row struct { - ID uint64 - Values []uint64 -} - -func (r *Row) Bitmap() []uint64 { - a := make([]uint64, rbf.ShardWidth/64) - for _, v := range r.Values { - a[v/64] |= 1 << (v % 64) - } - return a -} - -// Union returns the union of r and other's values. -func (r *Row) Union(other *Row) []uint64 { - m := make(map[uint64]struct{}) - for _, v := range r.Values { - m[v] = struct{}{} - } - for _, v := range other.Values { - m[v] = struct{}{} - } - - a := make([]uint64, 0, len(m)) - for v := range m { - a = append(a, v) - } - sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) - return a -} - -// Intersect returns the intersection of r & other's values. -func (r *Row) Intersect(other *Row) []uint64 { - m := make(map[uint64]struct{}) - for _, v := range r.Values { - m[v] = struct{}{} - } - - a := make([]uint64, 0) - used := make(map[uint64]struct{}) - for _, v := range other.Values { - if _, ok := used[v]; ok { - continue - } - if _, ok := m[v]; ok { - used[v] = struct{}{} - a = append(a, v) - } - } - sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) - return a -} - // QuickCheck executes fn multiple times with a different PRNG. func QuickCheck(t *testing.T, fn func(t *testing.T, rand *rand.Rand)) { for i := 0; i < *quickCheckN; i++ { diff --git a/rbf/tx.go b/rbf/tx.go index 17647caa3..690937691 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -201,6 +201,29 @@ func (tx *Tx) BitmapNames() ([]string, error) { return a, nil } +// BitmapExist returns true if bitmap exists. +func (tx *Tx) BitmapExists(name string) (bool, error) { + tx.mu.Lock() + defer tx.mu.Unlock() + return tx.bitmapExists(name) +} + +func (tx *Tx) bitmapExists(name string) (bool, error) { + if tx.db == nil { + return false, ErrTxClosed + } else if name == "" { + return false, ErrBitmapNameRequired + } + + // Read root records and find entry for bitmap. + records, err := tx.RootRecords() + if err != nil { + return false, err + } + _, ok := records.Get(name) + return ok, nil +} + // CreateBitmap creates a new empty bitmap with the given name. // Returns an error if the bitmap already exists. func (tx *Tx) CreateBitmap(name string) error { @@ -561,6 +584,31 @@ func (tx *Tx) Contains(name string, v uint64) (bool, error) { return c.Contains(v) } +// Depth returns the depth of the b-tree for a bitmap. +func (tx *Tx) Depth(name string) (int, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + + if tx.db == nil { + return 0, ErrTxClosed + } else if name == "" { + return 0, ErrBitmapNameRequired + } + + c, err := tx.cursor(name) + if err == ErrBitmapNotFound { + return 0, nil + } else if err != nil { + return 0, err + } + defer c.Close() + + if err := c.First(); err != nil { + return 0, err + } + return c.stack.top + 1, nil +} + // Cursor returns an instance of a cursor this bitmap. func (tx *Tx) Cursor(name string) (*Cursor, error) { tx.mu.RLock() @@ -669,6 +717,7 @@ func (tx *Tx) container(name string, key uint64) (*roaring.Container, error) { func (tx *Tx) PutContainer(name string, key uint64, ct *roaring.Container) error { tx.mu.Lock() defer tx.mu.Unlock() + return tx.putContainer(name, key, ct) } @@ -725,7 +774,6 @@ func (tx *Tx) removeContainer(name string, key uint64) error { if exact, err := c.Seek(key); err != nil || !exact { return err } - return c.deleteLeafCell(key) } @@ -1125,7 +1173,7 @@ func (tx *Tx) readPage(pgno uint32) (_ []byte, isHeap bool, err error) { // Verify page number requested is within current size of database. pageN := readMetaPageN(tx.meta[:]) - if pgno > pageN { + if pgno >= pageN { return nil, false, fmt.Errorf("rbf: page read out of bounds: pgno=%d max=%d", pgno, pageN-1) } @@ -1210,6 +1258,22 @@ func (tx *Tx) ContainerIterator(name string, key uint64) (citer roaring.Containe return &containerIterator{cursor: c}, exact, nil } +// Shared pool for in-memory database pages. +// These are used before being flushed to disk. +var containerFilterPool = &sync.Pool{} + +func getContainerFilter(c *Cursor, filter roaring.BitmapFilter, tx *Tx) *containerFilter { + existing := containerFilterPool.Get() + if existing == nil { + return &containerFilter{cursor: c, filter: filter, tx: tx} + } + f := existing.(*containerFilter) + f.cursor = c + f.filter = filter + f.tx = tx + return f +} + func (tx *Tx) ApplyFilter(name string, key uint64, filter roaring.BitmapFilter) (err error) { tx.mu.RLock() defer tx.mu.RUnlock() @@ -1225,7 +1289,7 @@ func (tx *Tx) ApplyFilter(name string, key uint64, filter roaring.BitmapFilter) if err != nil { return err } - f := containerFilter{cursor: c, filter: filter, tx: tx} + f := getContainerFilter(c, filter, tx) defer f.Close() return f.Apply() } @@ -1567,6 +1631,8 @@ type containerFilter struct { func (s *containerFilter) Close() { s.cursor.Close() + s.cursor = nil + containerFilterPool.Put(s) } func (s *containerFilter) Apply() (err error) { @@ -1932,6 +1998,7 @@ func (tx *Tx) Pages(pgnos []uint32) ([]Page, error) { // PageInfos returns meta data about all pages in the database. func (tx *Tx) PageInfos() ([]PageInfo, error) { var errorList ErrorList + infos := make([]PageInfo, tx.PageN()) // Read meta page info. diff --git a/rbf/tx_test.go b/rbf/tx_test.go index e9e493afc..1dc90a22b 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -2,6 +2,7 @@ package rbf_test import ( + "bytes" "encoding/binary" "fmt" "math/rand" @@ -443,7 +444,7 @@ func TestTx_DeallocateToFreeList(t *testing.T) { } } -func TestTx_Remove(t *testing.T) { +func TestTx_RemoveContainer(t *testing.T) { t.Parallel() db := MustOpenDB(t) @@ -541,6 +542,309 @@ func TestTx_AddRemove_Quick(t *testing.T) { }) } +func TestTx_Remove(t *testing.T) { + t.Run("FullContiguous", func(t *testing.T) { + if testing.Short() { + t.Skip("-short enabled, skipping") + } + + for _, bitN := range []uint64{1000, 100000, 2000000} { + t.Run(fmt.Sprint(bitN), func(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + // Add bits + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + for i := uint64(0); i < bitN; i++ { + if _, err := tx.Add("x", i); err != nil { + t.Fatalf("Add(%d) err=%q", i, err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + + // Remove bits + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + + for i := uint64(0); i < bitN; i++ { + if _, err := tx.Remove("x", i); err != nil { + t.Fatalf("Remove(%d) err=%q", i, err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + + // Verify that all bits have been removed. + tx := MustBegin(t, db, false) + defer tx.Rollback() + if n, err := tx.Count("x"); err != nil { + t.Fatal(err) + } else if got, want := n, uint64(0); got != want { + t.Fatalf("Count=%d, want %d", got, want) + } + }) + } + }) + + t.Run("PartialContiguous", func(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + // Add bits + const bitN = 100000 + const multiplier = 7 // space out bits so we span more containers + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + for i := uint64(0); i < bitN; i++ { + if _, err := tx.Add("x", i*multiplier); err != nil { + t.Fatalf("Add(%d) err=%q", i, err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + + // Remove some bits in small contiguous chunks. + var deleteN int + for i := uint64(bitN / 2); i < bitN; { + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + for j := uint64(0); j < 100; i, j = i+1, j+1 { + if n, err := tx.Remove("x", i*multiplier); err != nil || n != 1 { + t.Fatalf("Remove(%d)=(%v,%q)", i, n, err) + } + deleteN++ + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + } + + // Verify that we have the correct count afterward. + tx := MustBegin(t, db, false) + defer tx.Rollback() + if n, err := tx.Count("x"); err != nil { + t.Fatal(err) + } else if got, want := n, uint64(bitN-deleteN); got != want { + t.Fatalf("Count=%d, want %d", got, want) + } + }) + + t.Run("PartialNonContiguous", func(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + // Add bits + const bitN = 100000 + const multiplier = 7 // space out bits + bits := make([]uint64, 0, bitN) + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + for i := uint64(0); i < bitN; i++ { + if _, err := tx.Add("x", i*multiplier); err != nil { + t.Fatalf("Add(%d) err=%q", i, err) + } + bits = append(bits, i*multiplier) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + + // Remove some bits in small contiguous chunks. + var deleteN int + perm := rand.Perm(len(bits)) + for i := uint64(bitN / 2); i < bitN; { + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + for j := uint64(0); j < 100; i, j = i+1, j+1 { + value := bits[perm[i]] + if n, err := tx.Remove("x", value); err != nil || n != 1 { + t.Fatalf("Remove(%d)=(%v,%q)", value, n, err) + } + deleteN++ + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + } + + // Verify that we have the correct count afterward. + tx := MustBegin(t, db, false) + defer tx.Rollback() + if n, err := tx.Count("x"); err != nil { + t.Fatal(err) + } else if got, want := n, uint64(bitN-deleteN); got != want { + t.Fatalf("Count=%d, want %d", got, want) + } + }) + + t.Run("DeleteEmptyBitmap", func(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + // Create bitmap. + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } else if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + + // Remove bitmap. + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + if err := tx.DeleteBitmap("x"); err != nil { + t.Fatal(err) + } else if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + + // Ensure bitmap no longer exists. + tx := MustBegin(t, db, false) + defer tx.Rollback() + if exists, err := tx.BitmapExists("x"); err != nil { + t.Fatal(err) + } else if exists { + t.Fatal("expected bitmap to be removed") + } + }) + + t.Run("WithTreeDepth", func(t *testing.T) { + for depth := 1; depth <= 3; depth++ { + t.Run(fmt.Sprint(depth), func(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + // Create bitmap & insert until we hit a tree depth. + var bitN int + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } + for i := uint64(0); ; i++ { + if _, err := tx.Add("x", i<<16); err != nil { + t.Fatalf("Add(%d) err=%q", i<<16, err) + } + bitN++ + + if d, err := tx.Depth("x"); err != nil { + t.Fatal(err) + } else if d == depth { + break + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + + // Remove all bits in reverse order. + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + for i := bitN - 1; i >= 0; i-- { + if n, err := tx.Remove("x", uint64(i)<<16); err != nil || n != 1 { + t.Fatalf("Remove(%d)=(%v,%q)", uint64(i)<<16, n, err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + + // Ensure bitmap no longer exists. + tx := MustBegin(t, db, false) + defer tx.Rollback() + for i := uint64(0); i < uint64(bitN); i++ { + if ok, err := tx.Contains("x", i<<16); err != nil || ok { + t.Fatalf("Contains(%d)=(%v,%q)", i<<16, ok, err) + } + } + }) + } + }) + + t.Run("RollbackAfterDelete", func(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + + if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } else if err := tx.Commit(); err != nil { + t.Fatal(err) + } + }() + + // Add bits + const bitN = 1000 + for i := uint64(0); i < bitN; i++ { + func() { + tx := MustBegin(t, db, true) + defer tx.Rollback() + + if _, err := tx.Add("x", i<<16); err != nil { + t.Fatalf("Add(%d) err=%q", i<<16, err) + } + + // Only commit every other bit. + if i%2 == 1 { + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + } + }() + } + + // Verify that we have the correct count afterward. + tx := MustBegin(t, db, false) + defer tx.Rollback() + if n, err := tx.Count("x"); err != nil { + t.Fatal(err) + } else if got, want := n, uint64(bitN/2); got != want { + t.Fatalf("Count=%d, want %d", got, want) + } + }) +} + func TestTx_Multiple_CreateBitmap(t *testing.T) { rand := rand.New(rand.NewSource(0)) db := MustOpenDB(t) @@ -742,7 +1046,11 @@ func TestTx_DeleteBitmapsWithPrefix(t *testing.T) { t.Fatal(err) } } - checkInfos := func() { + var b bytes.Buffer + pBuf := func(msg string, args ...interface{}) (int, error) { + return fmt.Fprintf(&b, msg, args...) + } + checkInfos := func(pf func(string, ...interface{}) (int, error)) { tx := MustBegin(t, db, false) defer tx.Rollback() infos, err := tx.PageInfos() @@ -750,34 +1058,34 @@ func TestTx_DeleteBitmapsWithPrefix(t *testing.T) { for pgno, info := range infos { switch info := info.(type) { case *rbf.MetaPageInfo: - fmt.Printf("%-8d ", pgno) - fmt.Printf("%-10s ", "meta") - fmt.Printf("pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo) + pf("%-8d ", pgno) + pf("%-10s ", "meta") + pf("pageN=%d,walid=%d,rootrec=%d,freelist=%d\n", info.PageN, info.WALID, info.RootRecordPageNo, info.FreelistPageNo) case *rbf.RootRecordPageInfo: - fmt.Printf("%-8d ", pgno) - fmt.Printf("%-10s ", "rootrec") - fmt.Printf("next=%d\n", info.Next) + pf("%-8d ", pgno) + pf("%-10s ", "rootrec") + pf("next=%d\n", info.Next) case *rbf.LeafPageInfo: - fmt.Printf("%-8d ", pgno) - fmt.Printf("%-10s ", "leaf") - fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN) + pf("%-8d ", pgno) + pf("%-10s ", "leaf") + pf("flags=x%x,celln=%d\n", info.Flags, info.CellN) case *rbf.BranchPageInfo: - fmt.Printf("%-8d ", pgno) - fmt.Printf("%-10s ", "branch") - fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN) + pf("%-8d ", pgno) + pf("%-10s ", "branch") + pf("flags=x%x,celln=%d\n", info.Flags, info.CellN) case *rbf.BitmapPageInfo: - fmt.Printf("%-8d ", pgno) - fmt.Printf("%-10s ", "bitmap") - fmt.Printf("-\n") + pf("%-8d ", pgno) + pf("%-10s ", "bitmap") + pf("-\n") case *rbf.FreePageInfo: - fmt.Printf("%-8d ", pgno) - fmt.Printf("%-10s ", "free") - fmt.Printf("-\n") + pf("%-8d ", pgno) + pf("%-10s ", "free") + pf("-\n") default: t.Fatal(fmt.Sprintf("unexpected page info type %T", info)) @@ -806,19 +1114,19 @@ func TestTx_DeleteBitmapsWithPrefix(t *testing.T) { ifError(tx.Commit()) } - checkInfos() + checkInfos(pBuf) populate() - checkInfos() + checkInfos(pBuf) ifError(db.Check()) tx := MustBegin(t, db, true) tx.DeleteBitmapsWithPrefix(prefix) ifError(tx.Commit()) ifError(db.Check()) - checkInfos() + checkInfos(pBuf) populate() ifError(db.Check()) - checkInfos() + checkInfos(pBuf) } diff --git a/roaring/container_stash.go b/roaring/container_stash.go index e7e0f7cd3..fcff20daf 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -618,7 +618,7 @@ func (c *Container) setBitmap(bitmap []uint64) { } } if len(bitmap) != 1024 { - panic("illegal bitmap length") + panic(fmt.Sprintf("illegal bitmap length %v", len(bitmap))) } c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&bitmap[0])), bitmapN, bitmapN c.flags &^= flagPristine diff --git a/roaring/containers_test.go b/roaring/containers_test.go index e5d9fc2c4..5e170d4d1 100644 --- a/roaring/containers_test.go +++ b/roaring/containers_test.go @@ -176,6 +176,14 @@ func TestSliceContainers(t *testing.T) { }) } +func TestContainersFB1247(t *testing.T) { + bm := [bitmapN]uint64{0xF} + co := NewContainerBitmap(1, bm[:]) + co = co.bitmapToArray() + //should not panic + +} + func genRun(r *rand.Rand) Interval16 { gen: dat := r.Uint32() diff --git a/roaring/filter.go b/roaring/filter.go index 513f8e8f0..873337c23 100644 --- a/roaring/filter.go +++ b/roaring/filter.go @@ -584,6 +584,10 @@ type BitmapBitmapFilter struct { callback func(uint64) error } +func (b *BitmapBitmapFilter) SetCallback(cb func(uint64) error) { + b.callback = cb +} + func (b *BitmapBitmapFilter) ConsiderKey(key FilterKey, n int32) FilterResult { pos := key & keyMask if b.containers[pos] == nil || n == 0 { @@ -875,3 +879,136 @@ func ApplyFilterToIterator(filter BitmapFilter, iter ContainerIterator) error { } return nil } + +// BitmapBSICountFilter gives counts of values in each value-holding row +// of a BSI field, constrained by a filter. The first row of the data is +// taken to be an existence bit, which is intersected into the filter to +// constrain it, and the second is used as a sign bit. The rows after that +// are treated as value rows, and their counts of bits, overlapping with +// positive and negative bits in the sign rows, are returned to a callback +// function. +// +// The total counts of positions evaluated are returned with a row count +// of ^uint64(0) prior to row counts. +type BitmapBSICountFilter struct { + containers []*Container + positive []*Container + negative []*Container + nextOffsets []uint64 + count int32 + psum, nsum uint64 +} + +func (b *BitmapBSICountFilter) Total() (count int32, total int64) { + return b.count, int64(b.psum) - int64(b.nsum) +} + +func (b *BitmapBSICountFilter) ConsiderKey(key FilterKey, n int32) FilterResult { + pos := key & keyMask + if b.containers[pos] == nil || n == 0 { + return key.RejectUntilOffset(b.nextOffsets[pos]) + } + return key.NeedData() +} + +func (b *BitmapBSICountFilter) ConsiderData(key FilterKey, data *Container) FilterResult { + pos := key & keyMask + filter := b.containers[pos] + if filter == nil { + key.RejectUntilOffset(b.nextOffsets[pos]) + } + row := uint64(key >> rowExponent) // row count within the fragment + // How do we translate the filter and existence bit into actionable things? + // Assume the sign row is empty. We want positive values for anything in + // the intersection of the filter and the positive bits. If the sign row + // isn't empty, we want positive values for that intersection, less the + // sign row, and negative for the intersection of the filter/positive and + // the sign bits. So we can just stash the intermediate filter+existence + // as positive, then split it up if we have sign bits, which we often don't. + setup := false + switch row { + case 0: // existence bit + b.positive[pos] = intersect(b.containers[pos], data) + if b.positive[pos] == data { + b.positive[pos] = b.positive[pos].Clone() + } + b.count += int32(b.positive[pos].N()) + setup = true + case 1: // sign bit + // split into negative/positive components. doesn't affect total + // count. + b.negative[pos] = intersect(b.positive[pos], data) + if b.negative[pos] == data { + b.negative[pos] = b.negative[pos].Clone() + } + b.positive[pos] = difference(b.positive[pos], data) + setup = true + } + // if we were doing setup (first two rows), we're done + if setup { + return key.MatchOneUntilOffset(b.nextOffsets[pos]) + } + // helpful reminder: a nil container is a valid empty container, and + // intersectionCount knows this. + pcount := intersectionCount(b.positive[pos], data) + ncount := intersectionCount(b.negative[pos], data) + b.psum += (uint64(pcount) << (row - 2)) + b.nsum += (uint64(ncount) << (row - 2)) + return key.MatchOneUntilOffset(b.nextOffsets[pos]) +} + +// NewBitmapBSICountFilter creates a BitmapBSICountFilter, used for tasks +// like computing the sum of a BSI field matching a given filter. +// +// The input filter is assumed to represent one "row" of a shard's data, +// which is to say, a range of up to rowWidth consecutive containers starting +// at some multiple of rowWidth. We coerce that to the 0..rowWidth range +// because offset-within-row is what we care about. +func NewBitmapBSICountFilter(filter *Bitmap) *BitmapBSICountFilter { + containers := make([]*Container, rowWidth*3) + b := &BitmapBSICountFilter{ + containers: containers[:rowWidth], + positive: containers[rowWidth : rowWidth*2], + negative: containers[rowWidth*2 : rowWidth*3], + nextOffsets: make([]uint64, rowWidth), + } + if filter == nil { + for i := range b.containers { + b.containers[i] = NewContainerRun([]Interval16{{Start: 0, Last: 65535}}) + b.nextOffsets[i] = uint64(i+1) % rowWidth + } + return b + } + count := 0 + iter, _ := filter.Containers.Iterator(0) + last := uint64(0) + for iter.Next() { + k, v := iter.Value() + // Coerce container key into the 0-rowWidth range we'll be + // using to compare against containers within each row. + k = k & keyMask + b.containers[k] = v + last = k + count++ + } + // if there's only one container, we need to populate everything with + // its position. + if count == 1 { + for i := range b.containers { + b.nextOffsets[i] = last + } + } else { + // Point each container at the offset of the next valid container. + // With sparse bitmaps this will potentially make skipping faster. + for i := range b.containers { + if b.containers[i] != nil { + for int(last) != i { + b.nextOffsets[last] = uint64(i) + last = (last + 1) % rowWidth + } + } + } + } + + return b +} diff --git a/roaring/roaring.go b/roaring/roaring.go index f632aaca4..ffc0c3c3c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -675,6 +675,47 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { return output } +func (b *Bitmap) Hash(hash uint64) uint64 { + const ( + offset = 14695981039346656037 + prime = 1099511628211 + ) + if hash == 0 { + hash = uint64(offset) + } + + it, _ := b.Containers.Iterator(0) + for it.Next() { + ki, _ := it.Value() + hash ^= uint64(ki) + hash *= prime + } + + it, _ = b.Containers.Iterator(0) + for it.Next() { + _, ci := it.Value() + hash ^= 0 + hash *= prime + if ci.N() > 0 { + var bytes []byte + switch ci.typ() { + + case ContainerArray: + bytes = fromArray16(ci.array()) + case ContainerBitmap: + bytes = fromArray64(ci.bitmap()) + case ContainerRun: + bytes = fromInterval16(ci.runs()) + } + for _, b := range bytes { + hash ^= uint64(b) + hash *= prime + } + } + } + return hash +} + type mutableContainersIterator struct { c Containers @@ -3526,21 +3567,40 @@ func (c *Container) bitmapToArray() *Container { return c } bitmap := c.bitmap() - n := int32(0) - array := make([]uint16, c.N()) - for i, word := range bitmap { - for word != 0 { - t := word & -word - if roaringParanoia { - if n >= c.N() { - panic("bitmap has more bits set than container.n") + // FB-1247 adding an extra check just in case c.N proves to be unreliable + // TODO prove this has to be reliable + makeArray := func(bm []uint64, ar []uint16) ([]uint16, bool, int32) { + n := int32(0) + for i, word := range bm { + for word != 0 { + t := word & -word + if roaringParanoia { + if n >= c.N() { + panic("bitmap has more bits set than container.n") + } } + if n == int32(len(ar)) { + return ar, true, n + } + ar[n] = uint16((i*64 + int(popcount(t-1)))) + n++ + word ^= t } - array[n] = uint16((i*64 + int(popcount(t-1)))) - n++ - word ^= t } + return ar, false, n + } + array, fail, n := makeArray(bitmap, make([]uint16, c.N())) + if fail { + // the onlyreason we are here is because N was incorrect + // so we force a recount of N and try again + c.bitmapRepair() + array, fail, n = makeArray(bitmap, make([]uint16, c.N())) + if fail { + //this should not be able to happen under any circumstance + panic("bitmapToArray failure") + } + } if roaringParanoia { if n != c.N() { @@ -7488,3 +7548,13 @@ func (c *Container) Slice() (r []uint16) { } return r } + +func fromArray16(a []uint16) []byte { + return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2] +} +func fromArray64(a []uint64) []byte { + return (*[8192]byte)(unsafe.Pointer(&a[0]))[:8192:8192] +} +func fromInterval16(a []Interval16) []byte { + return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4] +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 7fc0e5bb1..00534cb69 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -4825,3 +4825,26 @@ func TestVariousBitmap(t *testing.T) { t.Fatal("nil AddN should be 0") } } +func TestBitmapHash(t *testing.T) { + a, b := NewContainerBitmapN(getFullBitmap(), MaxContainerVal+1), NewContainerBitmapN(getFullBitmap(), MaxContainerVal+1) + arr := NewContainerArray([]uint16{1, 2, 3, 5, 8}) + run := NewContainerRun([]Interval16{{Start: 0, Last: 32}}) + ba := NewBitmap() + bb := NewBitmap() + ba.Containers.Put(1, arr) + ba.Containers.Put(2, run) + ba.Containers.Put(101, a) + ba.Containers.Put(102, a) + + bb.Containers.Put(1, arr) + bb.Containers.Put(2, run) + bb.Containers.Put(101, b) + bb.Containers.Put(102, b) + if ba.Hash(0) != bb.Hash(0) { + t.Fatal("hash should be equal") + } + bb.Containers.Put(103, b) + if ba.Hash(0) == bb.Hash(0) { + t.Fatal("hash should be different") + } +} diff --git a/row.go b/row.go index b310d2ad0..1639f84ed 100644 --- a/row.go +++ b/row.go @@ -122,6 +122,15 @@ func (r *Row) ToTable() (*pb.TableResponse, error) { return pb.RowsToTable(r, n) } +// Hash calculate checksum code be useful in block hash join +func (r *Row) Hash() uint64 { + hash := uint64(0) + for i := range r.segments { + hash = r.segments[i].data.Hash(hash) + } + return hash +} + // ToRows implements the ToRowser interface. func (r *Row) ToRows(callback func(*pb.RowResponse) error) error { if len(r.Keys) > 0 { @@ -463,6 +472,11 @@ func (r *Row) MarshalJSON() ([]byte, error) { // Columns returns the columns in r as a slice of ints. func (r *Row) Columns() []uint64 { + // We occasionally hit cases where we want to call Columns on something + // that might not exist, but a nil slice would be fine. + if r == nil { + return nil + } a := make([]uint64, 0, r.Count()) for i := range r.segments { a = append(a, r.segments[i].Columns()...) diff --git a/server.go b/server.go index 531e56737..d68b0c40d 100644 --- a/server.go +++ b/server.go @@ -44,6 +44,7 @@ var _ broadcaster = &Server{} type Server struct { // nolint: maligned // Close management. wg sync.WaitGroup + muWG sync.Mutex closing chan struct{} // Internal @@ -75,6 +76,7 @@ type Server struct { // nolint: maligned antiEntropyInterval time.Duration metricInterval time.Duration diagnosticInterval time.Duration + ttlRemovalInterval time.Duration maxWritesPerRequest int confirmDownSleep time.Duration confirmDownRetries int @@ -99,6 +101,26 @@ func (s *Server) Holder() *Holder { return s.holder } +// addToWaitGroup adds to the server WaitGroup but makes sure the server isn't +// closing, and that the WaitGroup is not already waiting before it adds +func (s *Server) addToWaitGroup(delta int) bool { + select { + case <-s.closing: + return false + default: + s.muWG.Lock() + defer s.muWG.Unlock() + select { + case <-s.closing: + // if we're closing after having gotten the lock, stop!! + return false + default: + s.wg.Add(delta) + return true + } + } +} + // ServerOption is a functional option type for pilosa.Server type ServerOption func(s *Server) error @@ -146,6 +168,15 @@ func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { } } +// OptServerTtlRemovalInterval is a functional option on Server +// used to set the ttl removal interval. +func OptServerTtlRemovalInterval(interval time.Duration) ServerOption { + return func(s *Server) error { + s.ttlRemovalInterval = interval + return nil + } +} + // OptServerLongQueryTime is a functional option on Server // used to set long query duration. func OptServerLongQueryTime(dur time.Duration) ServerOption { @@ -412,6 +443,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { antiEntropyInterval: 0, metricInterval: 0, diagnosticInterval: 0, + ttlRemovalInterval: time.Hour, disCo: disco.NopDisCo, stator: disco.NopStator, @@ -590,7 +622,10 @@ func (s *Server) Open() error { // Start background process listening for translation // sync resets. - s.wg.Add(1) + if ok := s.addToWaitGroup(1); !ok { + return fmt.Errorf("closing server while opening server is NOT allowed") + } + go func() { defer s.wg.Done(); s.monitorResetTranslationSync() }() go func() { _ = s.translationSyncer.Reset() }() @@ -617,10 +652,13 @@ func (s *Server) Open() error { return errors.Wrap(err, "setting nodeState") } - s.wg.Add(3) + if ok := s.addToWaitGroup(4); !ok { + return fmt.Errorf("closing server while opening server is NOT allowed") + } go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() go func() { defer s.wg.Done(); s.monitorRuntime() }() go func() { defer s.wg.Done(); s.monitorDiagnostics() }() + go func() { defer s.wg.Done(); s.monitorTtl() }() toSend := func() []Message { s.holder.startMsgsMu.Lock() @@ -631,14 +669,18 @@ func (s *Server) Open() error { return toSend }() - s.wg.Add(1) + if ok := s.addToWaitGroup(1); !ok { + return fmt.Errorf("closing server while opening server is NOT allowed") + } go func() { defer s.wg.Done() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - s.wg.Add(1) + if ok := s.addToWaitGroup(1); !ok { + // the server is closing, stop!! + return + } go func() { defer s.wg.Done() defer cancel() @@ -716,11 +758,15 @@ func (s *Server) Close() error { case <-s.closing: return nil default: - errE := s.executor.Close() - + // get the muWG lock so that noone adds to the WaitGroup while it Waits + s.muWG.Lock() + defer s.muWG.Unlock() // Notify goroutines to stop. close(s.closing) s.wg.Wait() + + errE := s.executor.Close() + var errh, errd error var errhs error var errc error @@ -776,8 +822,11 @@ func (s *Server) monitorResetTranslationSync() { case <-s.closing: return case <-s.resetTranslationSyncCh: + if ok := s.addToWaitGroup(1); !ok { + // the server is closing!!! stop!! + return + } s.logger.Infof("holder translation sync beginning") - s.wg.Add(1) go func() { // Obtaining this lock ensures that there is only // one instance of resetTranslationSync() running @@ -793,6 +842,53 @@ func (s *Server) monitorResetTranslationSync() { } } +func (s *Server) monitorTtl() { + ctx := context.Background() + ticker := time.NewTicker(s.ttlRemovalInterval) + for { + select { + case <-s.closing: + return + case <-ticker.C: + s.TtlRemoval(ctx) + } + } +} + +func (s *Server) TtlRemoval(ctx context.Context) { + for _, index := range s.holder.Indexes() { + for _, field := range index.Fields() { + if field.Options().Type == "time" { + if field.Options().Ttl > 0 { + for _, view := range field.views() { + viewNames := strings.Split(view.name, "_") + if len(viewNames) >= 2 { + viewTime, err := timeOfView(view.name, false) + if err != nil { + s.logger.Printf("ttl parse view time: %s", err) + continue + } + timeSince := time.Since(viewTime) + + if timeSince >= field.Options().Ttl { + for _, shard := range field.AvailableShards(true).Slice() { + s.holder.txf.DeleteFragmentFromStore(index.Name(), field.Name(), view.name, shard, nil) + } + + err := s.defaultClient.api.DeleteView(ctx, index.Name(), field.Name(), view.name) + if err != nil { + s.logger.Errorf("ttl delete view: %s", err) + } + s.logger.Infof("ttl deleted view: %s", view.name) + } + } + } + } + } + } + } +} + func (s *Server) monitorAntiEntropy() { if s.antiEntropyInterval == 0 || s.cluster.ReplicaN <= 1 { return // anti entropy disabled diff --git a/server/config.go b/server/config.go index 50b8ad261..15fd7df0f 100644 --- a/server/config.go +++ b/server/config.go @@ -214,9 +214,6 @@ type Config struct { // LookupDBDSN is an external database to connect to for `ExternalLookup` queries. LookupDBDSN string `toml:"lookup-db-dsn"` - // The percentage of time spent recalculating the disk and memory usage cache. - UsageDutyCycle float64 `toml:"usage-duty-cycle"` - // Future flags are used to represent features or functionality which is not // yet the default behavior, but will be in a future release. Future struct { @@ -225,9 +222,6 @@ type Config struct { Rename bool `toml:"rename"` } `toml:"future"` - // Toggles /schema/details endpoint. If off, it returns empty. - SchemaDetailsOn bool `toml:"schema-details-on"` - Auth Auth } @@ -390,15 +384,9 @@ func NewConfig() *Config { c.Etcd.PeerCertFile = "" c.Etcd.PeerKeyFile = "" - // Disk and Memory Usage - c.UsageDutyCycle = 20.0 - // Future flags. c.Future.Rename = false - // Schema Details Toggle - c.SchemaDetailsOn = true - return c } diff --git a/server/grpc_test.go b/server/grpc_test.go index e2ecae03f..1124f332f 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -534,9 +534,10 @@ func TestQuerySQL(t *testing.T) { {"color", "[]string"}, {"height", "int64"}, {"score", "int64"}, + {"timestamp", "timestamp"}, }, rows: []row{ - {[]columnResponse{uint64(2), int64(16), []string{"blue"}, int64(30), int64(-8)}}, + {[]columnResponse{uint64(2), int64(16), []string{"blue"}, int64(30), int64(-8), "2011-01-02T12:32:00Z"}}, }, }, eq: equal, @@ -551,18 +552,19 @@ func TestQuerySQL(t *testing.T) { {"color", "[]string"}, {"height", "int64"}, {"score", "int64"}, + {"timestamp", "timestamp"}, }, rows: []row{ - {[]columnResponse{uint64(1), int64(27), []string{"blue"}, int64(20), int64(-10)}}, - {[]columnResponse{uint64(2), int64(16), []string{"blue"}, int64(30), int64(-8)}}, - {[]columnResponse{uint64(3), int64(19), []string{"red"}, int64(40), int64(6)}}, - {[]columnResponse{uint64(4), int64(27), []string{"green"}, int64(50), int64(0)}}, - {[]columnResponse{uint64(5), int64(16), []string{"blue"}, int64(60), int64(-2)}}, - {[]columnResponse{uint64(6), int64(34), []string{"blue"}, int64(70), int64(100)}}, - {[]columnResponse{uint64(7), int64(27), []string{"blue"}, int64(80), int64(0)}}, - {[]columnResponse{uint64(8), int64(16), []string{}, int64(90), int64(-13)}}, - {[]columnResponse{uint64(9), int64(16), []string{"red"}, int64(100), int64(80)}}, - {[]columnResponse{uint64(10), int64(31), []string{"red"}, int64(110), int64(-2)}}, + {[]columnResponse{uint64(1), int64(27), []string{"blue"}, int64(20), int64(-10), "2011-04-02T12:32:00Z"}}, + {[]columnResponse{uint64(2), int64(16), []string{"blue"}, int64(30), int64(-8), "2011-01-02T12:32:00Z"}}, + {[]columnResponse{uint64(3), int64(19), []string{"red"}, int64(40), int64(6), "2012-01-02T12:32:00Z"}}, + {[]columnResponse{uint64(4), int64(27), []string{"green"}, int64(50), int64(0), "2013-09-02T12:32:00Z"}}, + {[]columnResponse{uint64(5), int64(16), []string{"blue"}, int64(60), int64(-2), "2014-01-02T12:32:00Z"}}, + {[]columnResponse{uint64(6), int64(34), []string{"blue"}, int64(70), int64(100), "2010-05-02T12:32:00Z"}}, + {[]columnResponse{uint64(7), int64(27), []string{"blue"}, int64(80), int64(0), "2016-08-02T12:32:00Z"}}, + {[]columnResponse{uint64(8), int64(16), []string{}, int64(90), int64(-13), "2020-01-02T12:32:00Z"}}, + {[]columnResponse{uint64(9), int64(16), []string{"red"}, int64(100), int64(80), "2000-03-02T12:32:00Z"}}, + {[]columnResponse{uint64(10), int64(31), []string{"red"}, int64(110), int64(-2), "2018-01-02T12:32:00Z"}}, }, }, eq: equal, @@ -837,6 +839,64 @@ func TestQuerySQL(t *testing.T) { }, eq: equal, }, + { + // GroupBy(Rows(field='age'),Rows(field='height'),filter=Intersect(Row(timestamp>"2017-09-02T12:32:00Z"),Row(height>40))) + sql: "select age, height from grouper where timestamp > '2017-09-02T12:32:00Z' and height > 40 group by age, height", + exp: tableResponse{ + headers: []columnInfo{ + {"age", "int64"}, + {"height", "int64"}, + }, + rows: []row{ + {[]columnResponse{int64(16), int64(90)}}, + {[]columnResponse{int64(31), int64(110)}}, + }, + }, + eq: equalUnordered, + }, + { + // Extract(Union(Row(timestamp>"2017-09-02T12:32:00Z"),Row(height>90)),Rows(age), Rows(height)) + sql: "select age, height from grouper where timestamp > '2017-09-02T12:32:00Z' or height > 90", + exp: tableResponse{ + headers: []columnInfo{ + {"age", "int64"}, + {"height", "int64"}, + }, + rows: []row{ + {[]columnResponse{int64(16), int64(90)}}, + {[]columnResponse{int64(16), int64(100)}}, + {[]columnResponse{int64(31), int64(110)}}, + }, + }, + eq: equalUnordered, + }, + { + //Extract(Intersect(Row(timestamp>"2017-09-02T12:32:00Z"),Row(timestamp<"2019-09-02T12:32:00Z")),Rows(age), Rows(height)) + sql: "select age, height from grouper where timestamp > '2017-09-02T12:32:00Z' and timestamp < '2019-09-02T12:32:00Z'", + exp: tableResponse{ + headers: []columnInfo{ + {"age", "int64"}, + {"height", "int64"}, + }, + rows: []row{ + {[]columnResponse{int64(31), int64(110)}}, + }, + }, + eq: equalUnordered, + }, + { + //Distinct(Row(timestamp>"2019-09-02T12:32:00Z"), index='grouper',field='age') + sql: "select distinct age from grouper where timestamp > '2019-09-02T12:32:00Z'", + exp: tableResponse{ + headers: []columnInfo{ + {"age", "int64"}, + }, + rows: []row{ + {[]columnResponse{int64(16)}}, + }, + }, + eq: equalUnordered, + }, { sql: "show tables", exp: tableResponse{ @@ -865,6 +925,7 @@ func TestQuerySQL(t *testing.T) { {[]columnResponse{"color", "keyed-set"}}, {[]columnResponse{"height", "int"}}, {[]columnResponse{"score", "int"}}, + {[]columnResponse{"timestamp", "timestamp"}}, }, }, eq: equal, @@ -1007,6 +1068,10 @@ func TestQuerySQLWithError(t *testing.T) { sql: "select _id, age, field_not_found from grouper", err: pilosa.ErrFieldNotFound, }, + { + sql: "select age, color, count(*) from grouper group by field_not_found, age, color", + err: pilosa.ErrFieldNotFound, + }, } for i, test := range tests { @@ -1554,6 +1619,26 @@ func setUpTestQuerySQLUnary(ctx context.Context, t *testing.T) (gh *server.GRPCH t.Fatal(err) } } + m.MustCreateField(t, grouper.Name(), "timestamp", pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds)) + for id, timestamp := range map[int]string{ + 1: "2011-04-02T12:32:00Z", + 2: "2011-01-02T12:32:00Z", + 3: "2012-01-02T12:32:00Z", + 4: "2013-09-02T12:32:00Z", + 5: "2014-01-02T12:32:00Z", + 6: "2010-05-02T12:32:00Z", + 7: "2016-08-02T12:32:00Z", + 8: "2020-01-02T12:32:00Z", + 9: "2000-03-02T12:32:00Z", + 10: "2018-01-02T12:32:00Z", + } { + if _, err := gh.QueryPQLUnary(ctx, &pb.QueryPQLRequest{ + Index: grouper.Name(), + Pql: fmt.Sprintf("Set(%d, timestamp=\"%s\")", id, timestamp), + }); err != nil { + t.Fatal(err) + } + } // joiner joiner := m.MustCreateIndex(t, "joiner", pilosa.IndexOptions{TrackExistence: true}) @@ -1653,6 +1738,8 @@ func toTableResponse(resp *pb.TableResponse) tableResponse { tr.rows[i].columns[j] = v.Float64Val case *pb.ColumnResponse_DecimalVal: tr.rows[i].columns[j] = pql.NewDecimal(v.DecimalVal.Value, v.DecimalVal.Scale) + case *pb.ColumnResponse_TimestampVal: + tr.rows[i].columns[j] = v.TimestampVal default: tr.rows[i].columns[j] = nil } diff --git a/server/handler_test.go b/server/handler_test.go index 15712f414..2b17d7bd3 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -274,7 +274,7 @@ func TestHandler_Endpoints(t *testing.T) { } } - if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))); err != nil { + if f, err := i2.CreateFieldIfNotExists("f3", pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"), "0")); err != nil { t.Fatal(err) } else if _, err := f.SetBit(tx2, 0, 0, nil); err != nil { t.Fatal(err) @@ -302,8 +302,7 @@ func TestHandler_Endpoints(t *testing.T) { } var bodySchema pilosa.Schema - if err := json.Unmarshal(w.Body.Bytes(), - &bodySchema); err != nil { + if err := json.Unmarshal(w.Body.Bytes(), &bodySchema); err != nil { t.Fatalf("unexpected unmarshalling error: %v", err) } // DO NOT COMPARE `CreatedAt` - reset to 0 @@ -316,9 +315,8 @@ func TestHandler_Endpoints(t *testing.T) { // var targetSchema pilosa.Schema - target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":0},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1,"views":[{"name":"standard"}]}],"shardWidth":%[1]d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"cardinality":1,"views":[{"name":"standard"}]}],"shardWidth":%[1]d},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"cardinality":1,"views":[{"name":"standard"}]},{"name":"f1","options":{"type":"int","base":0,"bitDepth":0,"min":-100,"max":100,"keys":false,"foreignIndex":""},"cardinality":4,"views":[{"name":"bsig_f1"}]},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":0,"min":-10,"max":10,"keys":false},"cardinality":5,"views":[{"name":"bsig_f2"}]},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"cardinality":1,"views":[{"name":"standard"}]},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"cardinality":1,"views":[{"name":"standard"}]},{"name":"f5","options":{"type":"bool"},"cardinality":1,"views":[{"name":"standard"}]}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) - if err := json.Unmarshal([]byte(target), - &targetSchema); err != nil { + target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"views":[{"name":"standard"}]}],"shardWidth":%[1]d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false},"views":[{"name":"standard"}]}],"shardWidth":%[1]d},{"name":"i2","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":1000,"keys":false},"views":[{"name":"standard"}]},{"name":"f1","options":{"type":"int","base":0,"bitDepth":0,"min":-100,"max":100,"keys":false,"foreignIndex":""},"views":[{"name":"bsig_f1"}]},{"name":"f2","options":{"type":"decimal","base":0,"scale":1,"bitDepth":0,"min":-10,"max":10,"keys":false},"views":[{"name":"bsig_f2"}]},{"name":"f3","options":{"type":"time","timeQuantum":"YMDH","keys":false,"noStandardView":false},"views":[{"name":"standard"}]},{"name":"f4","options":{"type":"mutex","cacheType":"ranked","cacheSize":5000,"keys":false},"views":[{"name":"standard"}]},{"name":"f5","options":{"type":"bool"},"views":[{"name":"standard"}]}],"shardWidth":%[1]d}]}`, pilosa.ShardWidth) + if err := json.Unmarshal([]byte(target), &targetSchema); err != nil { t.Fatalf("unexpected unmarshalling error: %v", err) } @@ -327,38 +325,6 @@ func TestHandler_Endpoints(t *testing.T) { } }) - t.Run("SchemaDetailsOff", func(t *testing.T) { - err := cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(false)) - if err != nil { - t.Fatalf("setting schema details option") - } - - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema/details", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - - var bodySchema pilosa.Schema - if err := json.Unmarshal(w.Body.Bytes(), - &bodySchema); err != nil { - t.Fatalf("unexpected unmarshalling error: %v", err) - - } - for _, i := range bodySchema.Indexes { - for _, f := range i.Fields { - if f.Cardinality != nil { - t.Fatalf("expected nil cardinality, got: %v", *f.Cardinality) - } - } - } - - err = cmd.API.SetAPIOptions(pilosa.OptAPISchemaDetailsOn(true)) - if err != nil { - t.Fatalf("could not toggle schema details to on: %v", err) - } - }) - t.Run("Import", func(t *testing.T) { indexInfo, err := cmd.API.Schema(context.Background(), false) if err != nil { @@ -517,48 +483,6 @@ func TestHandler_Endpoints(t *testing.T) { } }) - // UI/usage returns disk and memory usage from a precalculated cache. - // Since the cache calculates the cache on server startup, and tests create indexes thereafter - // the cache initially has 0 indexes when the test suite is ran. Therefore, this test first - // resets the cache. - t.Run("UI/usage", func(t *testing.T) { - if cmd.API.ResetUsageCache() != nil { - t.Fatal(err) - } - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil)) - if w.Code != gohttp.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - nodeUsages := make(map[string]pilosa.NodeUsage) - if err := json.Unmarshal(w.Body.Bytes(), &nodeUsages); err != nil { - t.Fatalf("unmarshal") - } - - for _, nodeUsage := range nodeUsages { - if nodeUsage.Disk.TotalUse < 1 { - t.Fatalf("expected some disk use, got %d", nodeUsage.Disk.TotalUse) - } - if nodeUsage.Disk.Capacity < 1 { - t.Fatalf("expected some disk capacity, got %d", nodeUsage.Disk.Capacity) - } - if nodeUsage.Memory.TotalUse < 1 { - t.Fatalf("expected some memory use, got %d", nodeUsage.Memory.TotalUse) - } - if nodeUsage.Memory.Capacity < 1 { - t.Fatalf("expected some memory capacity, got %d", nodeUsage.Memory.Capacity) - } - numIndexes := len(nodeUsage.Disk.IndexUsage) - if numIndexes != 3 { - t.Fatalf("wrong length index usage list: expected %d, got %d", 3, numIndexes) - } - numFields := len(nodeUsage.Disk.IndexUsage["i1"].Fields) - if numFields != len(i1.Fields()) { - t.Fatalf("wrong length field usage list: expected %d, got %d", len(i1.Fields()), numFields) - } - } - }) - t.Run("UI/shard-distribution", func(t *testing.T) { // This tests the response structure, not the cluster behavior. w := httptest.NewRecorder() diff --git a/server/pg_test.go b/server/pg_test.go index 7250c6f7e..53f177191 100644 --- a/server/pg_test.go +++ b/server/pg_test.go @@ -29,7 +29,7 @@ func TestPostgresHandler(t *testing.T) { m.MustCreateField(t, "i", "keymutex", pilosa.OptFieldKeys(), pilosa.OptFieldTypeMutex(pilosa.CacheTypeNone, 0)) m.MustCreateField(t, "i", "int", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)) m.MustCreateField(t, "i", "decimal", pilosa.OptFieldTypeDecimal(2)) - m.MustCreateField(t, "i", "time", pilosa.OptFieldTypeTime("YMDH")) + m.MustCreateField(t, "i", "time", pilosa.OptFieldTypeTime("YMDH", "0")) m.MustCreateField(t, "i", "bool", pilosa.OptFieldTypeBool()) m.MustCreateIndex(t, "j", pilosa.IndexOptions{TrackExistence: true, Keys: true}) diff --git a/server/server.go b/server/server.go index 697816f16..5e368cfab 100644 --- a/server/server.go +++ b/server/server.go @@ -271,8 +271,6 @@ func (m *Command) Start() (err error) { } } - go m.API.RefreshUsageCache(m.Config.UsageDutyCycle) - _ = testhook.Opened(pilosa.NewAuditor(), m, nil) close(m.Started) return nil @@ -511,7 +509,6 @@ func (m *Command) SetupServer() error { m.API, err = pilosa.NewAPI( pilosa.OptAPIServer(m.Server), pilosa.OptAPIImportWorkerPoolSize(m.Config.ImportWorkerPoolSize), - pilosa.OptAPISchemaDetailsOn(m.Config.SchemaDetailsOn), ) if err != nil { return errors.Wrap(err, "new api") @@ -519,10 +516,6 @@ func (m *Command) SetupServer() error { // Tell server about its new API, which its client will need. m.Server.SetAPI(m.API) - if err != nil { - return errors.Wrap(err, "new grpc server") - } - var p authz.GroupPermissions if m.Config.Auth.Enable { m.Config.MustValidateAuth() diff --git a/server/server_test.go b/server/server_test.go index c1efbc8a7..093e919b5 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -760,7 +760,7 @@ func TestMain_ImportTimestamp(t *testing.T) { } // Create field. - if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"))); err != nil { + if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0")); err != nil { t.Fatal(err) } @@ -815,7 +815,7 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) { } // Create field. - if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), true)); err != nil { + if _, err := m.API.CreateField(context.Background(), indexName, fieldName, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMD"), "0", true)); err != nil { t.Fatal(err) } diff --git a/server_internal_test.go b/server_internal_test.go index da6d57578..b2e5d1116 100644 --- a/server_internal_test.go +++ b/server_internal_test.go @@ -35,3 +35,34 @@ func TestMonitorAntiEntropyZero(t *testing.T) { t.Fatalf("monitorAntiEntropy should have returned immediately with duration 0") } } + +func TestAddToWaitGroup(t *testing.T) { + // if this test times out / panics we have a problem, otherwise we're fine + td := t.TempDir() + cfg := &storage.Config{FsyncEnabled: false, Backend: storage.DefaultBackend} + s, err := NewServer(OptServerDataDir(td), OptServerStorageConfig(cfg)) + if err != nil { + t.Fatalf("making new server: %v", err) + } + + oks := make(chan bool, 10) + for i := 0; i < 10; i++ { + go func() { + oks <- s.addToWaitGroup(1) + time.Sleep(10 * time.Millisecond) + defer s.wg.Done() + }() + } + + for i := 0; i < 10; i++ { + ok := <-oks + if !ok { + t.Fatalf("unexpected close during WaitGroup add") + } + } + + s.Close() + if ok := s.addToWaitGroup(1); ok { + t.Fatalf("shouldn't be able to add while server is closing") + } +} diff --git a/server_test.go b/server_test.go new file mode 100644 index 000000000..487a39286 --- /dev/null +++ b/server_test.go @@ -0,0 +1,95 @@ +// Copyright 2022 Molecula Corp. All rights reserved. +package pilosa_test + +import ( + "context" + "fmt" + "reflect" + "sort" + "testing" + "time" + + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/test" +) + +func TestTtlRemoval(t *testing.T) { + + cluster := test.MustRunCluster(t, 1) + node := cluster.GetNode(0) + defer cluster.Close() + + // Create a client + client := node.Client() + + indexName := "i" + fieldName := "f" + + // Create indexes and field with ttl lasting 24 hours + if err := client.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{TrackExistence: true}); err != nil && err != pilosa.ErrIndexExists { + t.Fatalf("creating index, err: %v", err) + } else if err := client.CreateFieldWithOptions(context.Background(), indexName, fieldName, pilosa.FieldOptions{Ttl: time.Hour * 24, Type: pilosa.FieldTypeTime, TimeQuantum: "YMDH"}); err != nil { + t.Fatalf("creating field, err: %v", err) + } + + /* Set sample data 1 using this date: '2001-02-03T04:05', this will create these views: + - standard + - standard_2001 + - standard_200102 + - standard_20010203 + - standard_2001020304 + Since the sample date here is over 24 hours, all views except "standard" should get deleted + */ + _, err := client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(1, f=1, 2001-02-03T04:05)"}) + if err != nil { + t.Fatalf("setting sample data 1, err: %v", err) + } + + dateNow := time.Now() + dateNowString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour(), dateNow.Minute()) + /* Set sample data 2 using current time. + For example: current time is 2022-03-03T15:17 (also when the 24 hrs ttl countdown starts) will generate these views: + - standard_2022 -> gets converted to 2022_01_01, over 24 hours for ttl -> deleted + - standard_202203 -> gets converted to 2022_03_01, over 24 hours for ttl -> deleted + - standard_20220303 -> gets converted to 2022_03_03, within 24 hours -> keep + - standard_2022030315 -> gets converted to 2022_03_03 15:00, within 24 hours -> keep + */ + _, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(2, f=2, " + dateNowString + ")"}) + if err != nil { + t.Fatalf("setting sample data 2, err: %v", err) + } + + /* Set sample data 3 using yesterday's date + All views generated from this date should be deleted + */ + dateYesterday := time.Now().Add(-24*time.Hour + -1*time.Nanosecond) + dateYesterdayString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d", dateYesterday.Year(), dateYesterday.Month(), dateYesterday.Day(), dateYesterday.Hour(), dateYesterday.Minute()) + _, err = client.Query(context.Background(), indexName, &pilosa.QueryRequest{Index: indexName, Query: "Set(3, f=3, " + dateYesterdayString + ")"}) + if err != nil { + t.Fatalf("setting sample data 3, err: %v", err) + } + + node.Server.TtlRemoval(context.Background()) + + // Get all the views for given index + field + views, err := node.API.Views(context.Background(), indexName, fieldName) + if err != nil { + t.Fatal(err) + } + + expectedViewNames := []string{ + "standard", + "standard_" + fmt.Sprintf("%d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day()), + "standard_" + fmt.Sprintf("%d%02d%02d%02d", dateNow.Year(), dateNow.Month(), dateNow.Day(), dateNow.Hour()), + } + + var viewNames []string + for _, view := range views { + viewNames = append(viewNames, view.Name()) + } + sort.Strings(viewNames) + + if !reflect.DeepEqual(expectedViewNames, viewNames) { + t.Fatalf("after ttl removal, expected %v, but got %v", expectedViewNames, viewNames) + } +} diff --git a/sql/handler_test.go b/sql/handler_test.go index 6eaba476f..26ffce0ce 100644 --- a/sql/handler_test.go +++ b/sql/handler_test.go @@ -3,10 +3,13 @@ package sql_test import ( "context" + "math" "testing" + "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/sql" "github.com/molecula/featurebase/v3/test" + "vitess.io/vitess/go/vt/sqlparser" ) func TestHandler(t *testing.T) { @@ -28,3 +31,52 @@ func TestHandler(t *testing.T) { } } + +func TestSelectHandler_MapSelect(t *testing.T) { + cluster := test.MustRunCluster(t, 1) + defer cluster.Close() + api := cluster.GetNode(0).API + + if _, err := api.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil { + t.Fatal(err) + } else if _, err = api.CreateField(context.Background(), "i", "bytes", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { + t.Fatal(err) + } else if _, err = api.CreateField(context.Background(), "i", "duration_time", pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64)); err != nil { + t.Fatal(err) + } else if _, err = api.CreateField(context.Background(), "i", "timestamp", pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds)); err != nil { + t.Fatal(err) + } + + for _, tt := range []struct { + name string + input string + output string + }{ + { + name: "WhereTimestamp", + input: `SELECT * FROM i WHERE timestamp>"2000-01-01T00:00:00Z"`, + output: `Extract(Row(timestamp>"2000-01-01T00:00:00Z"),Rows(bytes),Rows(duration_time),Rows(timestamp))`, + }, + + { + name: "WhereTimestampWithSpaces", + input: `SELECT * FROM i WHERE timestamp > "2000-01-01T00:00:00Z"`, + output: `Extract(Row(timestamp>"2000-01-01T00:00:00Z"),Rows(bytes),Rows(duration_time),Rows(timestamp))`, + }, + } { + t.Run(tt.name, func(t *testing.T) { + query, err := sql.NewMapper().MapSQL(tt.input) + if err != nil { + t.Fatal(err) + } + + h := sql.NewSelectHandler(api) + mr, err := h.MapSelect(context.Background(), query.Statement.(*sqlparser.Select), query.Mask) + if err != nil { + t.Fatal(err) + } else if got, want := mr.Query, tt.output; got != want { + t.Fatalf("unexpected pql\npql: %s\nwant: %s", got, want) + } + }) + } +} diff --git a/sql/query.go b/sql/query.go index 0f4db98b7..80a1d2f49 100644 --- a/sql/query.go +++ b/sql/query.go @@ -15,32 +15,32 @@ const timeFormat = "2006-01-02T15:04" // LT creates a less than query. func LT(fieldName string, value interface{}) string { - return fmt.Sprintf("Row(%s<%s)", fieldName, intOrFloat(value)) + return fmt.Sprintf("Row(%s<%s)", fieldName, formatValue(value)) } // LTE creates a less than or equal query. func LTE(fieldName string, value interface{}) string { - return fmt.Sprintf("Row(%s<=%s)", fieldName, intOrFloat(value)) + return fmt.Sprintf("Row(%s<=%s)", fieldName, formatValue(value)) } // GT creates a greater than query. func GT(fieldName string, value interface{}) string { - return fmt.Sprintf("Row(%s>%s)", fieldName, intOrFloat(value)) + return fmt.Sprintf("Row(%s>%s)", fieldName, formatValue(value)) } // GTE creates a greater than or equal query. func GTE(fieldName string, value interface{}) string { - return fmt.Sprintf("Row(%s>=%s)", fieldName, intOrFloat(value)) + return fmt.Sprintf("Row(%s>=%s)", fieldName, formatValue(value)) } // Equals creates an equals query. func Equals(fieldName string, value interface{}) string { - return fmt.Sprintf("Row(%s=%s)", fieldName, intOrFloat(value)) + return fmt.Sprintf("Row(%s=%s)", fieldName, formatValue(value)) } // NotEquals creates a not equals query. func NotEquals(fieldName string, value interface{}) string { - return fmt.Sprintf("Row(%s!=%s)", fieldName, intOrFloat(value)) + return fmt.Sprintf("Row(%s!=%s)", fieldName, formatValue(value)) } // NotNull creates a not equal to null query. @@ -94,12 +94,18 @@ func Like(fieldName string, pattern string) string { // Between creates a between query. func Between(fieldName string, a interface{}, b interface{}) string { - return fmt.Sprintf("Row(%s >< [%s,%s])", fieldName, intOrFloat(a), intOrFloat(b)) + return fmt.Sprintf("Row(%s >< [%s,%s])", fieldName, formatValue(a), formatValue(b)) } // Distinct creates a Distinct query. -func Distinct(indexName, fieldName string) string { - return fmt.Sprintf("Distinct(Row(%s!=null),index='%s',field='%s')", fieldName, indexName, fieldName) +func Distinct(indexName, fieldName, rowCall string) string { + var b strings.Builder + fmt.Fprintf(&b, `Distinct(`) + if rowCall != "" { + fmt.Fprintf(&b, `%s, `, rowCall) + } + fmt.Fprintf(&b, `index='%s',field='%s')`, indexName, fieldName) + return b.String() } // RowDistinct creates a Distinct query with the given row filter. @@ -269,8 +275,10 @@ func formatIDKey(idKey interface{}) (string, error) { } } -func intOrFloat(value interface{}) string { +func formatValue(value interface{}) string { switch value.(type) { + case string: + return fmt.Sprintf("%q", value) case float64, float32: // In order to test expected values, we set the precision // to 8. TODO: It's likely we'll need to address this diff --git a/sql/reduce.go b/sql/reduce.go index 8de56c1f0..4aa9f09e7 100644 --- a/sql/reduce.go +++ b/sql/reduce.go @@ -347,6 +347,34 @@ func AssignHeaders(rowser pproto.ToRowser, headers ...Column) pproto.ToRowser { return &assignHeadersRowser{rowser, headers} } +type staticHeaderRowser struct { + rowser pproto.ToRowser + cols []Column +} + +func (a *staticHeaderRowser) ToRows(fn func(*pproto.RowResponse) error) error { + return a.rowser.ToRows(func(row *pproto.RowResponse) error { + var out pproto.RowResponse + + headers := make([]*pproto.ColumnInfo, len(row.Headers)) + for i := range row.Headers { + header := row.Headers[i] + header.Name = a.cols[i].Name() + headers[i] = header + } + out.Headers = headers + + out.Columns = row.Columns + + return fn(&out) + }) +} + +// StaticHeaders assigns fixed cols to a ToRowser. +func StaticHeaders(rowser pproto.ToRowser, cols ...Column) pproto.ToRowser { + return &staticHeaderRowser{rowser, cols} +} + var ( ErrIncompleteHeaders = errors.New("incomplete header assignment") ErrFieldNotInHeaders = errors.New("field not found in source header") diff --git a/sql/router.go b/sql/router.go index 8c6035dee..381b180af 100644 --- a/sql/router.go +++ b/sql/router.go @@ -29,6 +29,17 @@ func newRouter() *router { handlerSelectFieldsFromTableWhere{}, ) //// + selectRouter.addFilter( + NewQueryMask( + SelectPartDistinct|SelectPartField, + FromPartTable, + WherePartFieldCondition|WherePartMultiFieldCondition, + 0, + 0, + ), + []QueryMask{}, + handlerSelectDistinctFromTable{}, + ) selectRouter.addRoute("select distinct fld from tbl", handlerSelectDistinctFromTable{}) //// selectRouter.addFilter( @@ -58,7 +69,7 @@ func newRouter() *router { groupByOptional := NewQueryMask( SelectPartField|SelectPartFields|SelectPartCountStar|SelectPartSumField, FromPartTable, - WherePartFieldCondition, // TODO: this can probably handle fields as well + WherePartFieldCondition|WherePartMultiFieldCondition, GroupByPartField|GroupByPartFields, HavingPartCondition, ) diff --git a/sql/select.go b/sql/select.go index 3297ee0fc..b1f11e0d8 100644 --- a/sql/select.go +++ b/sql/select.go @@ -34,14 +34,14 @@ func (s *SelectHandler) Handle(ctx context.Context, mapped *MappedSQL) (pproto.T if !ok { return nil, fmt.Errorf("statement is not type select: %T", mapped.Statement) } - mr, err := s.mapSelect(ctx, stmt, mapped.Mask) + mr, err := s.MapSelect(ctx, stmt, mapped.Mask) if err != nil { return nil, errors.Wrap(err, "mapping select") } return s.execMappingResult(ctx, mr, mapped.SQL) } -func (s *SelectHandler) mapSelect(ctx context.Context, selectStmt *sqlparser.Select, qm QueryMask) (*MappingResult, error) { +func (s *SelectHandler) MapSelect(ctx context.Context, selectStmt *sqlparser.Select, qm QueryMask) (*MappingResult, error) { // Get the handler for this query mask. hndlr := s.router.handler(qm) if hndlr == nil { @@ -305,6 +305,15 @@ func (h handlerSelectDistinctFromTable) Apply(stmt *sqlparser.Select, qm QueryMa return nil, errors.New("distinct requires a valid field column") } + var wherePQL string + if stmt.Where != nil { + if wherePQL, err = extractWhere(index, stmt.Where.Expr); err != nil { + return nil, err + } + } else { + wherePQL = All() + } + limit, offset, hasLimit, hasOffset, err := extractLimitOffset(stmt) if err != nil { return nil, errors.Wrap(err, "extracting limit") @@ -315,22 +324,8 @@ func (h handlerSelectDistinctFromTable) Apply(stmt *sqlparser.Select, qm QueryMa return nil, errors.Wrap(err, "extracting order by") } - // Determine the type of the field needing distinct. - // If the pilosa field is type int, handle it as a Distinct() query. - // Otherwise, use Rows() - // TODO: ensure this works for all field types (bool, time, etc). - var qo string - if fieldCol.Field.Type() == pilosa.FieldTypeInt || fieldCol.Field.Type() == pilosa.FieldTypeTimestamp { - qo = Distinct(fieldCol.Field.Index(), fieldCol.Field.Name()) - } else { - if !qm.HasOrderBy() && limit > 0 { - if qo, err = RowsLimit(fieldCol.Field.Name(), int64(limit)); err != nil { - return nil, errors.Wrap(err, "creating Rows query") - } - } else { - qo = Rows(fieldCol.Field.Name()) - } - } + // We use a Distinct call instead of Rows as it supports filtering. + qo := Distinct(fieldCol.Field.Index(), fieldCol.Field.Name(), wherePQL) mr := &MappingResult{ IndexName: indexName, @@ -340,7 +335,7 @@ func (h handlerSelectDistinctFromTable) Apply(stmt *sqlparser.Select, qm QueryMa // Assign headers to the result. mr.addReducer(func(result pproto.ToRowser) pproto.ToRowser { - return AssignHeaders(result, selectFields...) + return StaticHeaders(result, selectFields...) }) if qm.HasOrderBy() { @@ -598,8 +593,7 @@ func (h handlerSelectGroupBy) Apply(stmt *sqlparser.Select, qm QueryMask, indexF rowsQueries := []string{} for _, fieldName := range groupByFieldNames { - field := index.Field(fieldName) - rowsQueries = append(rowsQueries, Rows(field.Name())) + rowsQueries = append(rowsQueries, Rows(fieldName)) } var wherePQL string @@ -796,7 +790,7 @@ func (h handlerSelectJoin) Apply(stmt *sqlparser.Select, qm QueryMask, indexFunc // Build the Distinct() portion of the query on the secondary. var distinctQry string if secondaryWhere == "" { - distinctQry = Distinct(secondaryField.Index(), secondaryField.Name()) + distinctQry = Distinct(secondaryField.Index(), secondaryField.Name(), "") } else { distinctQry = RowDistinct(secondaryField.Index(), secondaryField.Name(), secondaryWhere) } diff --git a/stats/stats_test.go b/stats/stats_test.go index 81b62a5f2..4515636c9 100644 --- a/stats/stats_test.go +++ b/stats/stats_test.go @@ -81,11 +81,6 @@ func TestStatsCount_TopN(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()} - hldr.SetBit("d", "f", 0, 0) - hldr.SetBit("d", "f", 0, 1) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2) - // Execute query. called := false hldr.Holder.Stats = &MockStats{ @@ -101,6 +96,12 @@ func TestStatsCount_TopN(t *testing.T) { called = true }, } + + hldr.SetBit("d", "f", 0, 0) + hldr.SetBit("d", "f", 0, 1) + hldr.SetBit("d", "f", 0, pilosa.ShardWidth) + hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2) + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil { t.Fatal(err) } diff --git a/stattx.go b/stattx.go index 4780f70bc..5ce265e90 100644 --- a/stattx.go +++ b/stattx.go @@ -159,6 +159,7 @@ const ( kOffsetRange kLast // mark the end, always keep this last. The following aren't tracked atm: kType + kRemoveChannel ) func (k kall) String() string { @@ -205,6 +206,8 @@ func (k kall) String() string { return "kLast" case kType: return "kType" + case kRemoveChannel: + return "kRemoveChannel" } vprint.PanicOn(fmt.Sprintf("unknown kall '%v'", int(k))) return "" @@ -221,6 +224,15 @@ func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring }() return c.b.NewTxIterator(index, field, view, shard) } +func (c *statTx) RemoveChannel(index, field, view string, shard uint64, a chan uint64, resChan chan countResults) { + me := kRemoveChannel + t0 := time.Now() + defer func() { + c.stats.add(me, time.Since(t0)) + }() + c.b.RemoveChannel(index, field, view, shard, a, resChan) + return +} func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { me := kImportRoaringBits diff --git a/time.go b/time.go index 1c479c996..945d87106 100644 --- a/time.go +++ b/time.go @@ -450,7 +450,7 @@ func timeOfView(v string, adj bool) (time.Time, error) { return time.Time{}, nil } - layout := "2006010203" + layout := "2006010215" timePart := viewTimePart(v) switch len(timePart) { diff --git a/translate.go b/translate.go index cc36ebbe2..9d5909f28 100644 --- a/translate.go +++ b/translate.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/molecula/featurebase/v3/ingest" + "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/topology" "github.com/pkg/errors" ) @@ -84,6 +85,8 @@ type TranslateStore interface { // TODO: refactor this interface; readonly shoul // It should read from the reader and replace the data store with // the read payload. ReadFrom(io.Reader) (int64, error) + + Delete(records *roaring.Bitmap) (Commitor, error) } // This implements ingest's key translator interface, which differs @@ -420,6 +423,16 @@ func (s *InMemTranslateStore) SetReadOnly(v bool) { defer s.mu.Unlock() s.readOnly = v } +func (s *InMemTranslateStore) Delete(records *roaring.Bitmap) (Commitor, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, id := range records.Slice() { + key := s.keysByID[id] + delete(s.keysByID, id) + delete(s.idsByKey, key) + } + return &NopCommitor{}, nil +} // FindKeys looks up the ID for each key. // Keys are not created if they do not exist. diff --git a/tx.go b/tx.go index 65a3e3c3f..7be5a54b7 100644 --- a/tx.go +++ b/tx.go @@ -18,13 +18,13 @@ const writable = true // // Within the fragment, the ckey or container-key is the uint64 that specifies // the high 48-bits of the roaring.Bitmap 64-bit space. -// The ckey is used to retreive a specific roaring.Container that -// is either a run, array, or raw-bitmap. The roaring.Container is the +// The ckey is used to retrieve a specific roaring.Container that +// is either a run, array, or raw bitmap. The roaring.Container is the // low 16-bits of the roaring.Bitmap space. Its size is at most // 8KB (2^16 bits / (8 bits / byte) == 8192 bytes). // // The grain of the transaction is guaranteed to be at least at the shard -// within one index. Therefore updates to the any of the fields within +// within one index. Therefore updates to any of the fields within // the same shard will be atomically visible only once the transaction commits. // Reads from another, concurrently open, transaction will not see updates // that have not been committed. @@ -34,7 +34,7 @@ type Tx interface { // Tx types at the top of txfactory.go Type() string - // Rollback must be called the end of read-only transactions. Either + // Rollback must be called at the end of read-only transactions. Either // Rollback or Commit must be called at the end of writable transactions. // It is safe to call Rollback multiple times, but it must be // called at least once to release resources. Any Rollback after @@ -50,10 +50,10 @@ type Tx interface { // Commit makes the updates in the Tx visible to subsequent transactions. Commit() error - // NewTxIterator returns it, a *roaring.Iterator whose it.Next() will + // NewTxIterator returns a *roaring.Iterator whose Next() method will // successively return each uint64 stored in the conceptual roaring.Bitmap // for the specified fragment. - NewTxIterator(index, field, view string, shard uint64) (it *roaring.Iterator) + NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator // ContainerIterator loops over the containers in the conceptual // roaring.Bitmap for the specified fragment. @@ -75,20 +75,21 @@ type Tx interface { // must copy it into some other memory. ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) - // RoaringBitmap retreives the roaring.Bitmap for the entire shard. + // RoaringBitmap retrieves the roaring.Bitmap for the entire shard. RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) // Container returns the roaring.Container for the given ckey - // (container-key or highbits), in the chosen fragment. + // (container-key or highbits) in the chosen fragment. Container(index, field, view string, shard uint64, ckey uint64) (*roaring.Container, error) - // PutContainer stores c under the given ckey (container-key), in the specified fragment. + // PutContainer stores c under the given ckey (container-key) in the specified fragment. PutContainer(index, field, view string, shard uint64, ckey uint64, c *roaring.Container) error - // RemoveContainer deletes the roaring.Container under the given ckey (container-key), + // RemoveContainer deletes the roaring.Container under the given ckey (container-key) // in the specified fragment. RemoveContainer(index, field, view string, shard uint64, ckey uint64) error + // Add adds the 'a' values to the Bitmap for the fragment. Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) // Remove removes the 'a' values from the Bitmap for the fragment. @@ -97,25 +98,46 @@ type Tx interface { // Contains tests if the uint64 v is stored in the fragment's Bitmap. Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) - // ForEach + // ForEach calls function `fn` on every value (bit set) in the Bitmap for + // the fragment. ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error - // ForEachRange + // ForEachRange calls function `fn` on every value (bit set) in the Bitmap for + // the fragment, limited to the [start, end) range. ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error - // Count + // Count returns the count of hot bits on the fragment. Count(index, field, view string, shard uint64) (uint64, error) - // Max + // Max returns the maximum value set in the Bitmap for the fragment. Max(index, field, view string, shard uint64) (uint64, error) - // Min + // Min returns the minimum value set in the Bitmap for the fragment. Min(index, field, view string, shard uint64) (uint64, bool, error) - // CountRange + // CountRange returns the count of hot bits in the [start, end) range on the + // fragment. CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) - // OffsetRange + // OffsetRange returns a *roaring.Bitmap containing the portion of the Bitmap for the fragment + // which is specified by a combination of (offset, [start, end)). + // + // start - The value at which to start reading. This must be the zero value + // of a container; i.e. [0, 65536, ...] + // end - The value at which to end reading. This must be the zero value + // of a container; i.e. [0, 65536, ...] + // offset - The number of positions to shift the resulting bitmap. This must + // be the zero value of a container; i.e. [0, 65536, ...] + // + // For example, if (index, field, view, shard) represents the following bitmap: + // [1, 2, 3, 65536, 65539] + // + // then the following results are achieved based on (offset, start, end): + // (0, 0, 131072) => [1, 2, 3, 65536, 65539] + // (0, 65536, 131072) => [0, 3] + // (65536, 65536, 131072) => [65536, 65539] + // (262144, 65536, 131072) => [262144, 262147] + // OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) // ImportRoaringBits does efficient bulk import using rit, a roaring.RoaringIterator. @@ -133,6 +155,7 @@ type Tx interface { GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) GetFieldSizeBytes(index, field string) (uint64, error) + RemoveChannel(index, field, view string, shard uint64, a chan uint64, resChan chan countResults) } // GenericApplyFilter implements ApplyFilter in terms of tx.ContainerIterator, diff --git a/tx_test.go b/tx_test.go index 80d72b51c..117198c94 100644 --- a/tx_test.go +++ b/tx_test.go @@ -242,5 +242,4 @@ func TestAPI_ImportAtomicRecord(t *testing.T) { if iraBit { PanicOn("IRA bit should have been cleared") } - } diff --git a/txfactory.go b/txfactory.go index 54dfa4390..4fddf79a9 100644 --- a/txfactory.go +++ b/txfactory.go @@ -4,7 +4,6 @@ package pilosa import ( "fmt" "os" - "path" "strings" "sync" @@ -471,192 +470,6 @@ func (f *TxFactory) DeleteFragmentFromStore( return f.dbPerShard.DeleteFragment(index, field, view, shard, frag) } -// IndexUsageDetails computes the sum of filesizes used by the node, broken down -// by index, field, fragments and keys. -func (f *TxFactory) IndexUsageDetails(isClosing func() bool) (map[string]IndexUsage, uint64, error) { - indexUsage := make(map[string]IndexUsage) - holderPath, err := expandDirName(f.holder.path) - if err != nil { - return indexUsage, 0, errors.Wrap(err, "expanding data directory") - } - indexesPath, err := expandDirName(f.holder.IndexesPath()) - if err != nil { - return indexUsage, 0, errors.Wrap(err, "expanding indexes directory") - } - - idxs := f.holder.Indexes() - - qcx := f.NewQcx() - defer qcx.Abort() - for _, idx := range idxs { - index := idx.name - indexPath := path.Join(indexesPath, index) - - // field usage - fieldUsages := make(map[string]FieldUsage) - fragmentsTotal := uint64(0) - fieldKeysTotal := uint64(0) - fieldMetaBytesTotal := uint64(0) - fieldsTotal := uint64(0) - flds := idx.Fields() - for _, fld := range flds { - field := fld.Name() - if field == "_keys" { - continue - } - fUsage, err := f.fieldUsage(indexPath, fld) - if err != nil { - return indexUsage, 0, errors.Wrapf(err, "getting disk usage for index (%s)", index) - } - - // non-roaring field usage - fragmentUsage := uint64(0) - - for _, shard := range fld.AvailableShards(true).Slice() { - if isClosing() { - return nil, 0, nil - } - if err := func() error { - tx, finisher, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) - if err != nil { - return errors.Wrap(err, "qcx.GetTx") - } - defer finisher(nil) - - fieldBytes, err := tx.GetFieldSizeBytes(index, field) - if err != nil { - return errors.Wrapf(err, "getting disk usage for non-roaring fragments (%s)", field) - } - fragmentUsage += fieldBytes - return nil - }(); err != nil { - return indexUsage, 0, err - } - } - - // add non-roaring to roaring - fUsage.Fragments += fragmentUsage - fUsage.Total += fragmentUsage - - // add to running total - fieldMetaBytesTotal += fUsage.Metadata - fieldKeysTotal += fUsage.Keys - fragmentsTotal += fUsage.Fragments - fieldsTotal += fUsage.Total - - fieldUsages[field] = fUsage - } - - // index metadata - indexMetaBytes, err := directoryUsage(indexPath, false) - if err != nil { - return indexUsage, 0, errors.Wrapf(err, "getting disk usage for index metadata (%s)", index) - } - - // index keys usage - indexKeysBytes := uint64(0) - if idx.keys { - keysPath := path.Join(indexPath, translateStoreDir) - indexKeysBytes, _ = directoryUsage(keysPath, true) // if directory doesn't exist, size = 0 - } - - indexUsage[index] = IndexUsage{ - Total: indexMetaBytes + indexKeysBytes + fieldsTotal, - Metadata: indexMetaBytes + fieldMetaBytesTotal, - IndexKeys: indexKeysBytes, - FieldKeysTotal: fieldKeysTotal, - Fragments: fragmentsTotal, - Fields: fieldUsages, - } - } - - // node metadata, e.g. id allocator - nodeMetaBytes, err := directoryUsage(holderPath, false) - if err != nil { - return indexUsage, 0, errors.Wrapf(err, "getting disk usage for node metadata") - } - - return indexUsage, nodeMetaBytes, nil -} - -// fieldUsage computes the sum of filesizes used by a field in -// the filesystem tree (roaring storage), broken down by keys and fragments. -func (f *TxFactory) fieldUsage(indexPath string, fld *Field) (FieldUsage, error) { - fieldUsage := FieldUsage{} - - field := fld.name - - // row keys - keysBytes := int64(0) - var err error - keysBytes, err = fileSize(fld.TranslateStorePath()) - if err != nil { - // if file doesn't exist, size = 0 - keysBytes = 0 - } - - // field metadata - fieldPath := path.Join(indexPath, FieldsDir, field) - metaBytes, err := directoryUsage(fieldPath, false) // this includes keys - if err != nil { - return fieldUsage, errors.Wrapf(err, "getting disk usage for field meta (%s)", field) - } - - // fragment data - viewsPath := path.Join(fieldPath, "views") - fragmentBytes := uint64(0) - if dirExists(viewsPath) { - fragmentBytes, err = directoryUsage(viewsPath, true) - if err != nil { - return fieldUsage, errors.Wrapf(err, "getting disk usage for field fragments (%s)", field) - } - } - - fieldUsage = FieldUsage{ - Total: metaBytes + fragmentBytes, // metaBytes includes keys - Metadata: metaBytes - uint64(keysBytes), - Fragments: fragmentBytes, - Keys: uint64(keysBytes), - } - - return fieldUsage, nil -} - -// NOTE: Go 1.16 introduced a new Readdir() method that is supposed to be more performant. -// Not yet upgraded b/c new method is not compatible with older versions of Go. -func directoryUsage(fname string, recursive bool) (uint64, error) { - if !dirExists(fname) { - return 0, errors.Errorf("directory does not exist (%s)", fname) - } - - var size uint64 - - dir, err := os.Open(fname) - if err != nil { - return 0, errors.Wrap(err, "opening data subdirectory") - } - defer dir.Close() - - files, err := dir.Readdir(-1) - if err != nil { - return 0, errors.Wrap(err, "reading data subdirectory") - } - - for _, file := range files { - if recursive && file.IsDir() { - sz, err := directoryUsage(path.Join(fname, file.Name()), true) - if err != nil { - return 0, err - } - size += sz - } else { - size += uint64(file.Size()) // NOTE this cast is safe for regular files, not necessarily others - } - } - - return size, nil -} - // CloseIndex is a no-op. This seems to be in place for debugging purposes. func (f *TxFactory) CloseIndex(idx *Index) error { return nil diff --git a/util.go b/util.go index eb9f958ce..ce4323ec3 100644 --- a/util.go +++ b/util.go @@ -4,8 +4,11 @@ package pilosa // util.go: a place for generic, reusable utilities. import ( + "fmt" "reflect" "time" + + "github.com/shirou/gopsutil/v3/mem" ) // LeftShifted16MaxContainerKey is 0xffffffffffff0000. It is similar @@ -54,3 +57,17 @@ func GetLoopProgress(start time.Time, now time.Time, iteration uint, total uint) func FormatTimestampNano(value, base int64, timeUnit string) string { return time.Unix(0, (value+base)*TimeUnitNanos(timeUnit)).UTC().Format(time.RFC3339Nano) } + +type MemoryUsage struct { + Capacity uint64 `json:"capacity"` + TotalUse uint64 `json:"totalUsed"` +} + +// GetMemoryUsage gets the memory usage +func GetMemoryUsage() (MemoryUsage, error) { + usage, err := mem.VirtualMemory() + if usage == nil || err != nil { + return MemoryUsage{}, fmt.Errorf("reading virtual memory: %v", err) + } + return MemoryUsage{Capacity: usage.Total, TotalUse: usage.Used}, nil +} diff --git a/util_test.go b/util_test.go index 870625ad8..9a1bfe7f9 100644 --- a/util_test.go +++ b/util_test.go @@ -90,3 +90,9 @@ func TestFormatTimestampNano(t *testing.T) { t.Fatal("Timestamp not formatted properly") } } + +func TestGetMemoryUsage(t *testing.T) { + if _, err := GetMemoryUsage(); err != nil { + t.Fatalf("unexpected error getting memory usage: %v", err) + } +} diff --git a/view.go b/view.go index d5e408810..927490a9f 100644 --- a/view.go +++ b/view.go @@ -307,6 +307,10 @@ func (v *view) recalculateCaches() { } } +func (v *view) Name() string { + return v.name +} + // CreateFragmentIfNotExists returns a fragment in the view by shard. func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { v.mu.Lock()