mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-11 23:31:03 +00:00
FB-1597: unifying idk and featurebase (#2160)
* unifying idk and featurebase: first pass * resolved conflict with master for gitignore & dockerignore * deleted binaries that were accidentally pushed to git * combined gitlab jobs for idk & featurebase * run go fmt for idk * updated ssh env variable, and made docker password variable in gitlab env variables * fixed typo assigning variable name * trying to fix docker login error * trying a different solution for docker password * pass registry * fixed docker login * updated paths for idk * exclude idk tests from featurebase test run * fix vendor error * update certificates * grpc needs to be in version 1.38 genproto, which is imported by big query updates the grpc version to 1.47.0 grpc 1.47.0 causes etcd to deadlock when calling etcd.Close() the fix is to have a replace in go.mod to specify a specific grpc version * run go mod tidy * go mod * run go mod tidy * exclude bigquery since it is causing issues and undo grpc replace in go.mod * fix grpc version * fix formatting error * update formatting * attempt to fix formatting * update path for code coverage * update to use current branch binaries, not master * fix for building idk - path updates * udpate path for binaries * update job dependecies * update docker idk tests to use the current branch registry * update stages for jobs * updated job dependencies * not allow idk s3 dump to fail since it is a dependency for integration tests * update dependecy for idk tests * update paths for idk build and code coverage * download featurebase binary from s3 * pass branch name to all setup scripts * change to current branch instead of master * updated sonarcloud * sonarcloud fix and branch name fix * trying to speed up pipeline run time * update stage * branch name fix + sonar cloud * sonarcloud
This commit is contained in:
souhailanoor
GitHub
parent
94a5721d05
commit
2bdf1f5081
239 changed files with 226860 additions and 197 deletions
|
|
@ -3,3 +3,5 @@ lattice/node_modules
|
|||
lattice/build
|
||||
statik/statik.go
|
||||
build
|
||||
testenv
|
||||
bin
|
||||
|
|
|
|||
43
.gitignore
vendored
43
.gitignore
vendored
|
|
@ -23,3 +23,46 @@ outputs.json
|
|||
builds/
|
||||
*.tfstate.backup
|
||||
.vscode
|
||||
|
||||
# copy of .gitignore from archived idk repo
|
||||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
*.exe
|
||||
*.test
|
||||
*.prof
|
||||
|
||||
vendor
|
||||
|
||||
.terraform
|
||||
terraform.tfstate*
|
||||
|
||||
bin
|
||||
build
|
||||
testenv
|
||||
.pulled
|
||||
|
||||
pilosa-sec-data-idk
|
||||
|
||||
.idea/
|
||||
tags.dot
|
||||
*.log
|
||||
*.swp
|
||||
|
|
|
|||
|
|
@ -3,20 +3,63 @@ include:
|
|||
- template: Security/License-Scanning.gitlab-ci.yml
|
||||
- template: Security/Dependency-Scanning.gitlab-ci.yml
|
||||
|
||||
gosec-sast:
|
||||
before_script:
|
||||
- export GOPRIVATE=github.com/molecula/*
|
||||
- apk add openssh-client
|
||||
- eval $(ssh-agent -s)
|
||||
- echo "$FB_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
||||
- mkdir -p ~/.ssh
|
||||
- chmod 700 ~/.ssh
|
||||
- git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/"
|
||||
- ssh-keygen -F github.com || echo "$SSH_KNOWN_HOSTS_HASHED" >> ~/.ssh/known_hosts
|
||||
- chmod 644 ~/.ssh/known_hosts
|
||||
|
||||
variables:
|
||||
GOVERSION: "1.17.8"
|
||||
|
||||
stages:
|
||||
- lint
|
||||
- pretest # run FB linux amd 64 because it is a dependency for idk tests
|
||||
- test
|
||||
- build
|
||||
- post build
|
||||
- integration
|
||||
- gauntlet
|
||||
- performance
|
||||
- post build
|
||||
- nonblocking
|
||||
- cleanup_build
|
||||
|
||||
.setup_ssh:
|
||||
before_script:
|
||||
- export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin
|
||||
- export GOPRIVATE=github.com/molecula/*
|
||||
|
||||
## Install ssh-agent if not already installed, it is required by Docker.
|
||||
## (change apt-get to yum if you use an RPM-based image)
|
||||
- "command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )"
|
||||
|
||||
## Run ssh-agent (inside the build environment)
|
||||
- eval $(ssh-agent -s)
|
||||
|
||||
## Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
|
||||
## We're using tr to fix line endings which makes ed25519 keys work
|
||||
## without extra base64 encoding.
|
||||
## https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
|
||||
- echo "$FB_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
||||
|
||||
## Create the SSH directory and give it the right permissions
|
||||
- mkdir -p ~/.ssh
|
||||
- chmod 700 ~/.ssh
|
||||
|
||||
- git config --global --get url."ssh://git@github.com/".insteadOf || git config --global --add url."ssh://git@github.com/".insteadOf "https://github.com/" || true
|
||||
## Set up known_hosts so we don't get prompted when there's no
|
||||
## human to answer the prompt (resulting in cryptic failures).
|
||||
## ssh-keygen -F checks whether github.com is in known_hosts and
|
||||
## handles HashKnownHosts appropriately.
|
||||
- ssh-keygen -F github.com || echo "$SSH_KNOWN_HOSTS_HASHED" >> ~/.ssh/known_hosts
|
||||
- chmod 644 ~/.ssh/known_hosts
|
||||
|
||||
smoke build:
|
||||
image: golang:$GOVERSION
|
||||
stage: lint
|
||||
|
|
@ -35,7 +78,7 @@ golangci-lint:
|
|||
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
|
||||
script:
|
||||
- echo "Checking for issues in new code"
|
||||
- golangci-lint run
|
||||
- golangci-lint run -v --timeout=8m
|
||||
|
||||
go mod tidy:
|
||||
stage: lint
|
||||
|
|
@ -46,8 +89,10 @@ go mod tidy:
|
|||
- go mod tidy
|
||||
- git diff --exit-code -- go.mod go.sum
|
||||
|
||||
### lattice, build for linux amd64 and build amd fb container are needed for IDK unit testing
|
||||
### running them at an earlier stage allows for decreasing run time.
|
||||
build lattice:
|
||||
stage: test
|
||||
stage: pretest
|
||||
image: node:14
|
||||
variables:
|
||||
CI: "false"
|
||||
|
|
@ -66,6 +111,41 @@ build lattice:
|
|||
paths:
|
||||
- lattice.tar.gz
|
||||
|
||||
build for linux amd64:
|
||||
stage: pretest
|
||||
image: golang:$GOVERSION
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
|
||||
script:
|
||||
- rm -r lattice
|
||||
- tar -xvf lattice.tar.gz
|
||||
- go get -v -u github.com/rakyll/statik
|
||||
- /go/bin/statik -src=lattice
|
||||
- GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
|
||||
- GOOS="linux" GOARCH="amd64" go build -o roaring-migrate_linux_amd64 ./cmd/roaring-migrate
|
||||
artifacts:
|
||||
paths:
|
||||
- featurebase_linux_amd64
|
||||
- roaring-migrate_linux_amd64
|
||||
needs:
|
||||
- job: build lattice
|
||||
|
||||
build amd container fb:
|
||||
stage: pretest
|
||||
tags:
|
||||
- shell
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
|
||||
before_script:
|
||||
- echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
|
||||
script:
|
||||
- tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
|
||||
- docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 -t $tag -f .gitlab/Dockerfile .
|
||||
- docker push $tag
|
||||
- echo Created docker featurebase image with tag "$tag"
|
||||
needs:
|
||||
- job: build for linux amd64
|
||||
|
||||
run jest tests:
|
||||
stage: test
|
||||
image: node:14
|
||||
|
|
@ -90,7 +170,7 @@ run go tests:
|
|||
retry: 1
|
||||
script:
|
||||
- echo "Running featurebase unit tests..."
|
||||
- go test -timeout=30m ./...
|
||||
- go test -v -timeout=30m $(go list ./... | grep -Ev idk)
|
||||
tags:
|
||||
- aws
|
||||
|
||||
|
|
@ -103,7 +183,7 @@ run go tests race:
|
|||
needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build
|
||||
script:
|
||||
- echo "Running featurebase race tests..."
|
||||
- go test -race -v -timeout=90m ./...
|
||||
- go test -race -v -timeout=90m $(go list ./... | grep -Ev idk)
|
||||
tags:
|
||||
- aws
|
||||
|
||||
|
|
@ -114,7 +194,7 @@ run go tests shardwidth22:
|
|||
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
|
||||
script:
|
||||
- echo "Running featurebase shardwidth22 tests..."
|
||||
- go test -timeout=30m -tags=shardwidth22 ./...
|
||||
- go test -timeout=30m -tags=shardwidth22 -v $(go list ./... | grep -Ev idk)
|
||||
tags:
|
||||
- aws
|
||||
|
||||
|
|
@ -129,8 +209,8 @@ run go tests future:
|
|||
retry: 1
|
||||
script:
|
||||
- echo "Running featurebase unit tests..."
|
||||
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
|
||||
- go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report.out
|
||||
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | grep -Ev 'idk' | paste -s -d, -)
|
||||
- go test -timeout=30m -json -coverprofile=coverage.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev idk) | tee test-report.out
|
||||
artifacts:
|
||||
paths:
|
||||
- coverage.out
|
||||
|
|
@ -146,8 +226,8 @@ run go tests future plg:
|
|||
retry: 1
|
||||
script:
|
||||
- echo "Running featurebase plg-specific unit tests..."
|
||||
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -)
|
||||
- go test -tags=plg -timeout=30m -coverprofile=coverage-plg.out -covermode=atomic -coverpkg=${PKG_LIST} ./... | tee test-report-plg.out
|
||||
- PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | grep -Ev 'idk' | paste -s -d, -)
|
||||
- go test -tags=plg -timeout=30m -coverprofile=coverage-plg.out -covermode=atomic -coverpkg=${PKG_LIST} $(go list ./... | grep -Ev idk) | tee test-report-plg.out
|
||||
artifacts:
|
||||
paths:
|
||||
- coverage-plg.out
|
||||
|
|
@ -155,6 +235,127 @@ run go tests future plg:
|
|||
tags:
|
||||
- aws
|
||||
|
||||
# idk tests
|
||||
run go tests idk race:
|
||||
extends:
|
||||
- .setup_ssh
|
||||
variables:
|
||||
USERNAME: fb-idk-access
|
||||
PROJECT: race_${CI_CONCURRENT_ID}
|
||||
stage: test
|
||||
retry: 1
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
|
||||
script:
|
||||
- echo "Running test-all-race"
|
||||
- cd ./idk/
|
||||
- command -v certstrap >/dev/null || go install github.com/square/certstrap@latest
|
||||
- echo $PROJECT
|
||||
- echo $DOCKER_PASSWORD | docker login registry.gitlab.com --username "$USERNAME" --password-stdin
|
||||
- BRANCH_NAME=${CI_COMMIT_BRANCH} make test-all-race
|
||||
after_script:
|
||||
- make save-pilosa-logs
|
||||
- make shutdown
|
||||
artifacts:
|
||||
paths:
|
||||
- ./idk/testdata/*_coverage.out
|
||||
- ./idk/testdata/*_report.out
|
||||
tags:
|
||||
- shell
|
||||
- aws
|
||||
needs:
|
||||
- job: build amd container fb
|
||||
|
||||
run go tests idk shard transactional:
|
||||
extends:
|
||||
- .setup_ssh
|
||||
variables:
|
||||
IDK_DEFAULT_SHARD_TRANSACTIONAL: 1
|
||||
USERNAME: fb-idk-access
|
||||
PROJECT: shardttrans_${CI_CONCURRENT_ID}
|
||||
stage: test
|
||||
retry: 1
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
|
||||
script:
|
||||
- echo "Running shard transactional tests"
|
||||
- cd ./idk/
|
||||
- command -v certstrap >/dev/null || go install github.com/square/certstrap@latest
|
||||
- echo $PROJECT
|
||||
- echo $DOCKER_PASSWORD | docker login registry.gitlab.com --username "$USERNAME" --password-stdin
|
||||
- BRANCH_NAME=${CI_COMMIT_BRANCH} make test-all
|
||||
after_script:
|
||||
- make save-pilosa-logs
|
||||
- make shutdown
|
||||
artifacts:
|
||||
paths:
|
||||
- ./idk/testdata/*_coverage.out
|
||||
- ./idk/testdata/*_report.out
|
||||
- ./idk/testdata/*_logs.txt
|
||||
tags:
|
||||
- shell
|
||||
- aws
|
||||
needs:
|
||||
- job: build amd container fb
|
||||
|
||||
run go tests idk 533:
|
||||
extends:
|
||||
- .setup_ssh
|
||||
variables:
|
||||
USERNAME: fb-idk-access
|
||||
PROJECT: test533_${CI_CONCURRENT_ID}
|
||||
stage: test
|
||||
retry: 1
|
||||
script:
|
||||
- echo "Running confluent 5.3.3 test-all"
|
||||
- cd ./idk/
|
||||
- command -v certstrap >/dev/null || go install github.com/square/certstrap@latest
|
||||
- echo $DOCKER_PASSWORD | docker login registry.gitlab.com --username "$USERNAME" --password-stdin
|
||||
- CONFLUENT_VERSION=5.3.3 BRANCH_NAME=${CI_COMMIT_BRANCH} make test-all
|
||||
after_script:
|
||||
- make save-pilosa-logs
|
||||
- make shutdown
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
|
||||
tags:
|
||||
- shell
|
||||
- aws
|
||||
artifacts:
|
||||
paths:
|
||||
- ./idk/testdata/*_coverage.out
|
||||
- ./idk/testdata/*_report.out
|
||||
needs:
|
||||
- job: build amd container fb
|
||||
|
||||
run go tests idk sasl:
|
||||
extends:
|
||||
- .setup_ssh
|
||||
variables:
|
||||
USERNAME: fb-idk-access
|
||||
PROJECT: sasl_${CI_CONCURRENT_ID}
|
||||
stage: test
|
||||
retry: 1
|
||||
script:
|
||||
- echo "Running test-all-kafka-sasl"
|
||||
- cd ./idk/
|
||||
- command -v certstrap >/dev/null || go install github.com/square/certstrap@latest
|
||||
- echo $DOCKER_PASSWORD | docker login registry.gitlab.com --username "$USERNAME" --password-stdin
|
||||
- BRANCH_NAME=${CI_COMMIT_BRANCH} make test-all-kafka-sasl
|
||||
after_script:
|
||||
- make save-pilosa-logs
|
||||
- make shutdown
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
|
||||
tags:
|
||||
- shell
|
||||
- aws
|
||||
artifacts:
|
||||
paths:
|
||||
- ./idk/testdata/*_coverage.out
|
||||
- ./idk/testdata/*_report.out
|
||||
needs:
|
||||
- job: build amd container fb
|
||||
|
||||
upload to sonarcloud:
|
||||
stage: integration
|
||||
image: sonarsource/sonar-scanner-cli:4.6
|
||||
|
|
@ -164,28 +365,17 @@ upload to sonarcloud:
|
|||
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
|
||||
script:
|
||||
- sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=coverage*.out,results/coverage*out -Dsonar.go.tests.reportPaths=test-report*.out -Dsonar.javascript.lcov.reportPaths=lattice/coverage/lcov.info
|
||||
- cd ./idk
|
||||
- ls ./testdata
|
||||
- sonar-scanner -Dsonar.projectKey=molecula_featurebase -Dsonar.organization=molecula -Dsonar.sources=. -Dsonar.host.url=https://sonarcloud.io -Dsonar.go.coverage.reportPaths=testdata/*coverage.out -Dsonar.go.tests.reportPaths=testdata/*report.out -Dsonar.coverage.exclusions=**/*_test.go -Dsonar.cpd.exclusions=**/*_test.go
|
||||
needs:
|
||||
- job: run go tests future plg
|
||||
- job: run go tests future
|
||||
- job: run jest tests
|
||||
- job: clustertests
|
||||
|
||||
build for linux amd64:
|
||||
stage: build
|
||||
image: golang:$GOVERSION
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
|
||||
script:
|
||||
- rm -r lattice
|
||||
- tar -xvf lattice.tar.gz
|
||||
- go get -v -u github.com/rakyll/statik
|
||||
- /go/bin/statik -src=lattice
|
||||
- GOOS="linux" GOARCH="amd64" make build FLAGS="-o featurebase_linux_amd64"
|
||||
- GOOS="linux" GOARCH="amd64" go build -o roaring-migrate_linux_amd64 ./cmd/roaring-migrate
|
||||
artifacts:
|
||||
paths:
|
||||
- featurebase_linux_amd64
|
||||
- roaring-migrate_linux_amd64
|
||||
- job: run go tests idk race
|
||||
- job: run go tests idk sasl
|
||||
- job: run go tests idk 533
|
||||
|
||||
build for linux arm64:
|
||||
stage: build
|
||||
|
|
@ -377,22 +567,6 @@ package plg for linux arm64:
|
|||
|
||||
# end PLG
|
||||
|
||||
build amd container fb:
|
||||
stage: build
|
||||
needs:
|
||||
- "build for linux amd64"
|
||||
tags:
|
||||
- shell
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
|
||||
before_script:
|
||||
- echo "${DOCKER_DEPLOY_TOKEN}" | docker login -u ${DOCKER_DEPLOY_USER} --password-stdin ${CI_REGISTRY}
|
||||
script:
|
||||
- tag=${CI_REGISTRY_IMAGE}/featurebase:linux-amd64-${CI_COMMIT_REF_SLUG}
|
||||
- docker build --build-arg GO_VERSION=$GOVERSION --build-arg ARCH=amd64 -t $tag -f .gitlab/Dockerfile .
|
||||
- docker push $tag
|
||||
- echo Created docker featurebase image with tag "$tag"
|
||||
|
||||
build arm container fb:
|
||||
stage: build
|
||||
needs:
|
||||
|
|
@ -409,6 +583,128 @@ build arm container fb:
|
|||
- docker push $tag
|
||||
- echo Created docker featurebase image with tag "$tag"
|
||||
|
||||
### start idk builds ###
|
||||
# building them all serially because otherwise you get container name conflicts.
|
||||
idk build_all:
|
||||
stage: build
|
||||
variables:
|
||||
BUILD_NAME: build_${CI_COMMIT_SHA}_${CI_CONCURRENT_ID}
|
||||
extends:
|
||||
- .setup_ssh
|
||||
tags:
|
||||
- shell
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
|
||||
script:
|
||||
- cd ./idk/
|
||||
- make docker-build GOOS="linux" GOARCH="amd64" BUILD_CGO=1
|
||||
- make docker-build GOOS="darwin" GOARCH="amd64"
|
||||
- make docker-build GOOS="darwin" GOARCH="arm64"
|
||||
artifacts:
|
||||
paths:
|
||||
- ./idk/build/*
|
||||
|
||||
idk build_linux_arm64:
|
||||
stage: build
|
||||
extends:
|
||||
- .setup_ssh
|
||||
tags:
|
||||
- shell-arm64
|
||||
rules:
|
||||
- if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
|
||||
script:
|
||||
- cd ./idk/
|
||||
- make docker-build GOOS="linux" GOARCH="arm64" BUILD_CGO=1 BUILD_NAME="linux-arm64"
|
||||
artifacts:
|
||||
paths:
|
||||
- ./idk/build/*
|
||||
|
||||
# building them all serially because otherwise you get container name conflicts.
|
||||
# only do containers on default branch
|
||||
idk package_docker_all:
|
||||
stage: build
|
||||
extends:
|
||||
- .setup_ssh
|
||||
tags:
|
||||
- shell
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
|
||||
script:
|
||||
- cd ./idk/
|
||||
- make docker GOOS="linux" GOARCH="amd64"
|
||||
- make docker GOOS="linux" GOARCH="arm64"
|
||||
- make docker GOOS="darwin" GOARCH="amd64"
|
||||
- make docker GOOS="darwin" GOARCH="arm64"
|
||||
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
|
||||
- make docker-tag-push GOOS="linux" GOARCH="amd64"
|
||||
- make docker-tag-push GOOS="linux" GOARCH="arm64"
|
||||
- make docker-tag-push GOOS="darwin" GOARCH="amd64"
|
||||
- make docker-tag-push GOOS="darwin" GOARCH="arm64"
|
||||
needs:
|
||||
- job: idk build_all
|
||||
- job: idk build_linux_arm64
|
||||
|
||||
idk s3 dump:
|
||||
stage: post build
|
||||
extends:
|
||||
- .setup_ssh
|
||||
allow_failure: false
|
||||
variables:
|
||||
PROFILE: "service-fb-ci"
|
||||
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
|
||||
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
|
||||
tags:
|
||||
- shell
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG == null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
|
||||
script:
|
||||
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
|
||||
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
|
||||
- aws configure set region "us-east-2"
|
||||
- aws configure set aws_profile $PROFILE
|
||||
- aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/ --recursive
|
||||
- aws s3 cp ./idk/build/ s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/ --recursive
|
||||
needs:
|
||||
- job: idk build_all
|
||||
- job: idk build_linux_arm64
|
||||
|
||||
idk s3 dump tag:
|
||||
stage: post build
|
||||
extends:
|
||||
- .setup_ssh
|
||||
variables:
|
||||
PROFILE: "service-fb-ci"
|
||||
AWS_SSH_PRIVATE_KEY: $AWS_FBCI_SSH_KEY
|
||||
AWS_ACCESS_KEY_ID: $AWS_FBCI_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY: $AWS_FBCI_SECRET_ACCESS_KEY
|
||||
LOCATION: molecula-artifact-storage/idk/_tags
|
||||
tags:
|
||||
- shell
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG != null && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
|
||||
script:
|
||||
- aws configure set aws_access_key_id $AWS_FBCI_ACCESS_KEY_ID
|
||||
- aws configure set aws_secret_access_key $AWS_FBCI_SECRET_ACCESS_KEY
|
||||
- aws configure set region "us-east-2"
|
||||
- aws configure set aws_profile $PROFILE
|
||||
- |
|
||||
for goos in "darwin" "linux"; do
|
||||
for goarch in "amd64" "arm64"; do
|
||||
dir=idk-${CI_COMMIT_TAG}-${goos}-${goarch}
|
||||
echo "Directory ${dir}"
|
||||
mkdir ${dir}
|
||||
mv ./idk/build/idk-${goos}-${goarch}/molecula-consumer-* ${dir}/
|
||||
tar cvzf ${dir}.tar.gz ${dir}
|
||||
aws s3 cp ${dir} s3://${LOCATION}/${CI_COMMIT_TAG}/${dir}/ --recursive
|
||||
aws s3 cp ${dir}.tar.gz s3://${LOCATION}/${CI_COMMIT_TAG}/
|
||||
done
|
||||
done
|
||||
needs:
|
||||
- job: idk build_all
|
||||
- job: idk build_linux_arm64
|
||||
### end idk builds ###
|
||||
|
||||
# clustertests doesn't run in docker, and requires several things to be set up on the runner to work:
|
||||
# 1. Install Go, make sure it's on the path
|
||||
# 2. Make sure "make" is installed
|
||||
|
|
@ -502,16 +798,18 @@ smoke test auth:
|
|||
- export PATH=$PATH:/usr/local/go/bin
|
||||
- TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-auth-smoke-$CI_JOB_ID"
|
||||
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
|
||||
# download datagen for FB-1270 repro test. TODO replace w/ locally built datagen once we merge IDK into FB
|
||||
- aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64
|
||||
- chmod +x ./datagen_linux_arm64
|
||||
# download datagen for FB-1270 repro test.
|
||||
- aws s3 cp s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64
|
||||
- aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
|
||||
- chmod +x ./datagen_linux_arm64 ./featurebase_linux_arm64
|
||||
script:
|
||||
- ./qa/scripts/auth-smoke/setup.sh
|
||||
- ./qa/scripts/auth-smoke/setup.sh $CI_COMMIT_BRANCH
|
||||
- ./qa/scripts/auth-smoke/test.sh
|
||||
after_script:
|
||||
- ./qa/scripts/auth-smoke/teardown.sh
|
||||
needs:
|
||||
- job: build for linux arm64
|
||||
- job: s3 dump
|
||||
- job: idk s3 dump
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
|
|
@ -559,17 +857,19 @@ smoke test:
|
|||
- export PATH=$PATH:/usr/local/go/bin
|
||||
- TF_VAR_cluster_prefix="pipeline-$CI_PIPELINE_ID-smoke-$CI_JOB_ID"
|
||||
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
|
||||
# download datagen for FB-1270 repro test. TODO replace w/ locally built datagen once we merge IDK into FB
|
||||
- aws s3 cp s3://molecula-artifact-storage/idk/master/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64
|
||||
- chmod +x ./datagen_linux_arm64
|
||||
# download datagen for FB-1270 repro test.
|
||||
- aws s3 cp s3://molecula-artifact-storage/idk/${CI_COMMIT_BRANCH}/_latest/idk-linux-arm64/datagen ./datagen_linux_arm64
|
||||
- aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
|
||||
- chmod +x ./datagen_linux_arm64 ./featurebase_linux_arm64
|
||||
script:
|
||||
- ./qa/scripts/setupSmokeTest.sh
|
||||
- ./qa/scripts/setupSmokeTest.sh $CI_COMMIT_BRANCH
|
||||
- ./qa/scripts/testSmokeTest.sh
|
||||
- ./qa/scripts/bug_repro_tests.sh
|
||||
after_script:
|
||||
- ./qa/scripts/teardownSmokeTest.sh
|
||||
needs:
|
||||
- job: build for linux arm64
|
||||
- job: s3 dump
|
||||
- job: idk s3 dump
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
|
|
@ -628,19 +928,19 @@ tremor-delete-test:
|
|||
script:
|
||||
- echo "Start CloudFormation deploy..."
|
||||
- >
|
||||
aws cloudformation deploy
|
||||
--stack-name $CF_STACK_NAME
|
||||
--template-file ./qa/cft/FeatureBaseClusterCFTTremor.yaml
|
||||
--parameter-overrides
|
||||
FBInstanceCount=$FB_INSTANCE_COUNT
|
||||
EBSVolumeSize=1200
|
||||
TestName=$CI_JOB_NAME
|
||||
TestID=$CI_JOB_ID
|
||||
Name=$CF_VAR_cluster_prefix
|
||||
Prefix=$CF_VAR_cluster_prefix
|
||||
TestCommitSHA=$CI_COMMIT_SHA
|
||||
--capabilities CAPABILITY_NAMED_IAM
|
||||
--profile $PROFILE
|
||||
aws cloudformation deploy
|
||||
--stack-name $CF_STACK_NAME
|
||||
--template-file ./qa/cft/FeatureBaseClusterCFTTremor.yaml
|
||||
--parameter-overrides
|
||||
FBInstanceCount=$FB_INSTANCE_COUNT
|
||||
EBSVolumeSize=1200
|
||||
TestName=$CI_JOB_NAME
|
||||
TestID=$CI_JOB_ID
|
||||
Name=$CF_VAR_cluster_prefix
|
||||
Prefix=$CF_VAR_cluster_prefix
|
||||
TestCommitSHA=$CI_COMMIT_SHA
|
||||
--capabilities CAPABILITY_NAMED_IAM
|
||||
--profile $PROFILE
|
||||
- echo "CloudFormation deploy done."
|
||||
- echo "Start configuring and start featurebase..."
|
||||
- ./qa/scripts/setupTremorDeleteCFT.sh $CF_STACK_NAME $PROFILE $REPLICA_COUNT $FB_INSTANCE_COUNT $CI_COMMIT_BRANCH
|
||||
|
|
@ -660,6 +960,9 @@ tremor-delete-test:
|
|||
- echo "Deleting stack after test, stack name = ${CF_STACK_NAME}"
|
||||
- aws cloudformation delete-stack --stack-name $CF_STACK_NAME --profile $PROFILE --retain-resources DeploymentEC2Role
|
||||
- echo "Delete stack complete!"
|
||||
needs:
|
||||
- job: s3 dump
|
||||
- job: idk s3 dump
|
||||
|
||||
samsung-gauntlet:
|
||||
stage: gauntlet
|
||||
|
|
@ -710,13 +1013,18 @@ samsung-gauntlet:
|
|||
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
|
||||
- export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
|
||||
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
|
||||
- aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
|
||||
- chmod +x ./featurebase_linux_arm64
|
||||
script:
|
||||
- ./qa/scripts/setupSamsungGauntlet.sh
|
||||
- ./qa/scripts/setupSamsungGauntlet.sh $CI_COMMIT_BRANCH
|
||||
- ./qa/scripts/testSamsungGauntlet.sh
|
||||
after_script:
|
||||
- ./qa/scripts/teardownSamsungGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
|
||||
- export INSTANCE_ID=$(cat instance_id)
|
||||
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
|
||||
needs:
|
||||
- job: s3 dump
|
||||
- job: idk s3 dump
|
||||
|
||||
backup-restore-gauntlet:
|
||||
stage: gauntlet
|
||||
|
|
@ -767,15 +1075,18 @@ backup-restore-gauntlet:
|
|||
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
|
||||
- export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
|
||||
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
|
||||
- aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
|
||||
- chmod +x ./featurebase_linux_arm64
|
||||
script:
|
||||
- ./qa/scripts/setupBackupRestoreGauntlet.sh
|
||||
- ./qa/scripts/testBackupRestoreGauntlet.sh
|
||||
- ./qa/scripts/setupBackupRestoreGauntlet.sh $CI_COMMIT_BRANCH
|
||||
- ./qa/scripts/testBackupRestoreGauntlet.sh $CI_COMMIT_BRANCH
|
||||
after_script:
|
||||
- ./qa/scripts/teardownBackupRestoreGauntlet.sh || true # leaving dangling resources is better than dangling ASG instances that can't be terminated
|
||||
- export INSTANCE_ID=$(cat instance_id)
|
||||
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
|
||||
needs:
|
||||
- build for linux arm64
|
||||
- job: s3 dump
|
||||
- job: idk s3 dump
|
||||
|
||||
s3 dump:
|
||||
stage: post build
|
||||
|
|
@ -808,7 +1119,7 @@ s3 dump:
|
|||
- aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/featurebase_darwin_arm64
|
||||
- aws s3 cp featurebase_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_darwin_arm64
|
||||
- aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/roaring-migrate_darwin_arm64
|
||||
- aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_arm64
|
||||
- aws s3 cp roaring-migrate_darwin_arm64 s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/roaring-migrate_darwin_arm64
|
||||
- aws s3 cp single_node_featurebase_linux_amd64 s3://molecula-artifact-storage/single_node_featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/single_node_featurebase_linux_amd64
|
||||
- aws s3 cp single_node_featurebase_linux_amd64 s3://molecula-artifact-storage/single_node_featurebase/${CI_COMMIT_BRANCH}/_latest/single_node_featurebase_linux_amd64
|
||||
- aws s3 cp single_node_featurebase_linux_arm64 s3://molecula-artifact-storage/single_node_featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/single_node_featurebase_linux_arm64
|
||||
|
|
@ -816,7 +1127,7 @@ s3 dump:
|
|||
- aws s3 cp single_node_featurebase_darwin_amd64 s3://molecula-artifact-storage/single_node_featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/single_node_featurebase_darwin_amd64
|
||||
- aws s3 cp single_node_featurebase_darwin_amd64 s3://molecula-artifact-storage/single_node_featurebase/${CI_COMMIT_BRANCH}/_latest/single_node_featurebase_darwin_amd64
|
||||
- aws s3 cp single_node_featurebase_darwin_arm64 s3://molecula-artifact-storage/single_node_featurebase/${CI_COMMIT_BRANCH}/${CI_COMMIT_SHORT_SHA}/single_node_featurebase_darwin_arm64
|
||||
- aws s3 cp single_node_featurebase_darwin_arm64 s3://molecula-artifact-storage/single_node_featurebase/${CI_COMMIT_BRANCH}/_latest/single_node_featurebase_darwin_arm64
|
||||
- aws s3 cp single_node_featurebase_darwin_arm64 s3://molecula-artifact-storage/single_node_featurebase/${CI_COMMIT_BRANCH}/_latest/single_node_featurebase_darwin_arm64
|
||||
needs:
|
||||
- job: build for darwin amd64
|
||||
- job: build for darwin arm64
|
||||
|
|
@ -889,15 +1200,18 @@ perf_delete:
|
|||
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
|
||||
- export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
|
||||
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
|
||||
- aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
|
||||
- chmod +x ./featurebase_linux_arm64
|
||||
script:
|
||||
- ./qa/scripts/perf/delete/deleteSetup.sh
|
||||
- ./qa/scripts/perf/delete/deleteTest.sh
|
||||
- ./qa/scripts/perf/delete/deleteSetup.sh $CI_COMMIT_BRANCH
|
||||
- ./qa/scripts/perf/delete/deleteTest.sh $CI_COMMIT_BRANCH
|
||||
after_script:
|
||||
- ./qa/scripts/perf/delete/deleteTeardown.sh || true
|
||||
- export INSTANCE_ID=$(cat instance_id)
|
||||
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --no-protected-from-scale-in --profile $INFRA_PROFILE
|
||||
needs:
|
||||
- job: build for linux arm64
|
||||
- job: s3 dump
|
||||
- job: idk s3 dump
|
||||
|
||||
s3 dump tag:
|
||||
stage: post build
|
||||
|
|
@ -992,4 +1306,6 @@ cleanup_build_job:
|
|||
- aws configure set aws_profile $FBCI_PROFILE
|
||||
- ./qa/scripts/gitlabCleanupBuild.sh
|
||||
when: always
|
||||
|
||||
needs:
|
||||
- job: smoke test
|
||||
- job: smoke test auth
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
variables:
|
||||
GOVERSION: "1.17.8"
|
||||
|
||||
|
||||
stages:
|
||||
- performance
|
||||
|
||||
|
|
@ -34,7 +34,7 @@ perf_able:
|
|||
- aws configure set region "us-east-2" --profile $INFRA_PROFILE
|
||||
- echo $AWS_FBCI_SSH_KEY > gitlab-featurebase-ci.pem
|
||||
- chmod 400 gitlab-featurebase-ci.pem
|
||||
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
|
||||
- "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
|
||||
- eval $(ssh-agent -s)
|
||||
- mkdir -p ~/.ssh
|
||||
- echo $AWS_FBCI_SSH_KEY > /root/.ssh/gitlab-featurebase-ci.pem
|
||||
|
|
@ -50,8 +50,10 @@ perf_able:
|
|||
- echo "Cluster Prefix --> $TF_VAR_cluster_prefix"
|
||||
- export INSTANCE_ID=$(curl --silent --fail "http://169.254.169.254/latest/meta-data/instance-id" | tee instance_id)
|
||||
- aws autoscaling set-instance-protection --instance-ids "$INSTANCE_ID" --auto-scaling-group-name $ASG_NAME --protected-from-scale-in --profile $INFRA_PROFILE
|
||||
- aws s3 cp s3://molecula-artifact-storage/featurebase/${CI_COMMIT_BRANCH}/_latest/featurebase_linux_arm64 ./
|
||||
- chmod +x ./featurebase_linux_arm64
|
||||
script:
|
||||
- ./qa/scripts/perf/able/ableSetup.sh
|
||||
- ./qa/scripts/perf/able/ableSetup.sh $CI_COMMIT_BRANCH
|
||||
- ./qa/scripts/perf/able/ableTest.sh
|
||||
after_script:
|
||||
- ./qa/scripts/perf/able/ableTeardown.sh || true
|
||||
|
|
|
|||
89
go.mod
89
go.mod
|
|
@ -1,20 +1,30 @@
|
|||
module github.com/molecula/featurebase/v3
|
||||
|
||||
replace github.com/go-avro/avro => github.com/pilosa/avro v0.0.0-20200626214113-bc1bf9fd41c1
|
||||
|
||||
require (
|
||||
github.com/CAFxX/gcnotifier v0.0.0-20220409005548-0153238b886a
|
||||
github.com/DataDog/datadog-go v4.8.3+incompatible
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
|
||||
github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2
|
||||
github.com/aws/aws-sdk-go v1.42.39
|
||||
github.com/beevik/ntp v0.3.0
|
||||
github.com/benbjohnson/immutable v0.3.0
|
||||
github.com/buger/jsonparser v1.1.1
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/confluentinc/confluent-kafka-go v1.9.0
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/felixge/fgprof v0.9.1
|
||||
github.com/denisenkom/go-mssqldb v0.11.0
|
||||
github.com/felixge/fgprof v0.9.2
|
||||
github.com/getsentry/sentry-go v0.13.0
|
||||
github.com/glycerine/vprint v0.0.0-20200730000117-76cea49a68ea
|
||||
github.com/go-avro/avro v0.0.0-20171219232920-444163702c11
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/go-test/deep v1.0.7
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/google/go-cmp v0.5.7
|
||||
github.com/google/go-cmp v0.5.8
|
||||
github.com/gorilla/handlers v1.3.0
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/gorilla/securecookie v1.1.1
|
||||
|
|
@ -25,18 +35,21 @@ require (
|
|||
github.com/opentracing/opentracing-go v1.2.0
|
||||
github.com/pelletier/go-toml v1.9.5
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/prometheus/client_golang v1.11.1
|
||||
github.com/prometheus/client_golang v1.12.2
|
||||
github.com/prometheus/client_model v0.2.0
|
||||
github.com/prometheus/prom2json v1.3.0
|
||||
github.com/prometheus/prom2json v1.3.1
|
||||
github.com/rakyll/statik v0.1.7
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shirou/gopsutil/v3 v3.21.9
|
||||
github.com/spf13/cobra v1.1.3
|
||||
github.com/segmentio/kafka-go v0.4.29
|
||||
github.com/shirou/gopsutil/v3 v3.22.5
|
||||
github.com/spf13/cobra v1.2.1
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/spf13/viper v1.7.1
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/spf13/viper v1.8.1
|
||||
github.com/stretchr/testify v1.7.1
|
||||
github.com/uber/jaeger-client-go v2.25.0+incompatible
|
||||
github.com/zeebo/blake3 v0.1.1
|
||||
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
|
||||
github.com/zeebo/blake3 v0.2.3
|
||||
go.etcd.io/bbolt v1.3.6
|
||||
go.etcd.io/etcd v3.3.27+incompatible
|
||||
go.etcd.io/etcd/api/v3 v3.5.4
|
||||
|
|
@ -45,22 +58,28 @@ require (
|
|||
go.etcd.io/etcd/server/v3 v3.5.4
|
||||
golang.org/x/exp v0.0.0-20201008143054-e3b2a7f2fdc7
|
||||
golang.org/x/mod v0.5.1
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
|
||||
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
|
||||
google.golang.org/grpc v1.38.0
|
||||
gopkg.in/DataDog/dd-trace-go.v1 v1.38.1
|
||||
gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
modernc.org/mathutil v1.0.0
|
||||
modernc.org/strutil v1.0.0
|
||||
sigs.k8s.io/yaml v1.2.0
|
||||
vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/jaffee/commandeer v0.5.0
|
||||
github.com/linkedin/goavro/v2 v2.11.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/DataDog/datadog-go/v5 v5.1.0 // indirect
|
||||
github.com/DataDog/gostackparse v0.5.0 // indirect
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||
github.com/StackExchange/wmi v1.2.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
|
|
@ -70,10 +89,12 @@ require (
|
|||
github.com/dustin/go-humanize v1.0.0 // indirect
|
||||
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/go-ole/go-ole v1.2.5 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/btree v1.0.1 // indirect
|
||||
github.com/google/pprof v0.0.0-20210423192551-a2663126120b // indirect
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
|
|
@ -83,30 +104,35 @@ require (
|
|||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jonboulle/clockwork v0.3.0 // indirect
|
||||
github.com/json-iterator/go v1.1.11 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.15.1 // indirect
|
||||
github.com/magiconair/properties v1.8.1 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/magiconair/properties v1.8.5 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.1 // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.14 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/common v0.26.0 // indirect
|
||||
github.com/prometheus/procfs v0.6.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/prometheus/common v0.33.0 // indirect
|
||||
github.com/prometheus/procfs v0.7.3 // indirect
|
||||
github.com/rs/cors v1.8.2 // indirect
|
||||
github.com/sirupsen/logrus v1.7.0 // indirect
|
||||
github.com/soheilhy/cmux v0.1.5 // indirect
|
||||
github.com/spf13/afero v1.2.2 // indirect
|
||||
github.com/spf13/cast v1.3.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.0.0 // indirect
|
||||
github.com/spf13/afero v1.6.0 // indirect
|
||||
github.com/spf13/cast v1.3.1 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/stretchr/objx v0.2.0 // indirect
|
||||
github.com/subosito/gotenv v1.2.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.9 // indirect
|
||||
github.com/tklauser/numcpus v0.3.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.10 // indirect
|
||||
github.com/tklauser/numcpus v0.4.0 // indirect
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
|
||||
github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
go.etcd.io/etcd/client/v2 v2.305.4 // indirect
|
||||
go.etcd.io/etcd/pkg/v3 v3.5.4 // indirect
|
||||
go.etcd.io/etcd/raft/v3 v3.5.4 // indirect
|
||||
|
|
@ -127,16 +153,13 @@ require (
|
|||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
google.golang.org/appengine v1.6.6 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
gopkg.in/ini.v1 v1.51.1 // indirect
|
||||
google.golang.org/protobuf v1.28.0 // indirect
|
||||
gopkg.in/ini.v1 v1.62.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||
nhooyr.io/websocket v1.8.6 // indirect
|
||||
sigs.k8s.io/yaml v1.2.0 // indirect
|
||||
)
|
||||
|
||||
go 1.17
|
||||
|
|
|
|||
260
go.sum
260
go.sum
|
|
@ -10,8 +10,15 @@ cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO
|
|||
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
|
||||
cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
|
||||
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
|
||||
cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms=
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
|
||||
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
|
||||
cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
|
||||
cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
|
||||
cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=
|
||||
cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
|
||||
|
|
@ -30,6 +37,7 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
|
|||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
|
||||
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
|
||||
|
|
@ -60,8 +68,8 @@ github.com/DataDog/gostackparse v0.5.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZ
|
|||
github.com/DataDog/sketches-go v1.0.0 h1:chm5KSXO7kO+ywGWJ0Zs6tdmWU8PBXSbywFVciL6BG4=
|
||||
github.com/DataDog/sketches-go v1.0.0/go.mod h1:O+XkJHWk9w4hDwY2ZUDU31ZC9sNYlYo8DiFsxjYeo1k=
|
||||
github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgRRqzCuPshRkQ7I=
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
|
||||
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
|
||||
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
|
||||
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
|
||||
|
|
@ -78,8 +86,6 @@ github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqR
|
|||
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
|
||||
github.com/Shopify/sarama v1.22.0/go.mod h1:lm3THZ8reqBDBQKQyb5HB3sY1lKp3grEbQ81aWSgPp4=
|
||||
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
|
||||
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
|
||||
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
|
||||
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
|
||||
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
|
||||
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
|
||||
|
|
@ -89,6 +95,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
|
|||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2 h1:090cWAt7zsbdvRegKCBVwcCTghjxhUh1PK2KNSq82vw=
|
||||
github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2/go.mod h1:c5eyz5amZqTKvY3ipqerFO/74a/8CYmXOahSr40c+Ww=
|
||||
github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
|
|
@ -103,6 +111,8 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ
|
|||
github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
|
||||
github.com/aws/aws-sdk-go v1.42.39 h1:6Lso73VoCI8Zmv3zAMv4BNg2gHAKNOlbLv1s/ew90SI=
|
||||
github.com/aws/aws-sdk-go v1.42.39/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
|
||||
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
|
||||
github.com/aws/aws-sdk-go-v2 v1.0.0/go.mod h1:smfAbmpW+tcRVuNUjo3MOArSZmW72t62rkCzc2i0TWM=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.0.0/go.mod h1:WysE/OpUgE37tjtmtJd8GXgT8s1euilE5XtUkRNUQ1w=
|
||||
|
|
@ -127,6 +137,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
|
|||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
|
||||
github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
|
||||
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
|
||||
|
|
@ -154,6 +165,7 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
|
|||
github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
|
||||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
|
||||
|
|
@ -166,6 +178,8 @@ github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u9
|
|||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
|
||||
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
|
||||
github.com/confluentinc/confluent-kafka-go v1.4.0/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg=
|
||||
github.com/confluentinc/confluent-kafka-go v1.9.0 h1:d1k62oAuQVxgdMdiDQnpkABbtIWTBwXHpDcyGQUw5QQ=
|
||||
github.com/confluentinc/confluent-kafka-go v1.9.0/go.mod h1:WDFs+KlhHITEoCzEfHSNgj5aP7vjajyYbZpvTEGs1sE=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ=
|
||||
|
|
@ -193,6 +207,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
|||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
|
||||
github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI=
|
||||
github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
|
||||
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I=
|
||||
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE=
|
||||
|
|
@ -220,6 +235,7 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s
|
|||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
|
|
@ -230,8 +246,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
|
|||
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
|
||||
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/felixge/fgprof v0.9.1 h1:E6FUJ2Mlv043ipLOCFqo8+cHo9MhQ203E2cdEK/isEs=
|
||||
github.com/felixge/fgprof v0.9.1/go.mod h1:7/HK6JFtFaARhIljgP2IV8rJLIoHDoOYoUphsnGvqxE=
|
||||
github.com/felixge/fgprof v0.9.2 h1:tAMHtWMyl6E0BimjVbFt7fieU6FpjttsZN7j0wT5blc=
|
||||
github.com/felixge/fgprof v0.9.2/go.mod h1:+VNi+ZXtHIQ6wIw6bUT8nXQRefQflWECoFyRealT5sg=
|
||||
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c=
|
||||
github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
|
||||
|
|
@ -257,6 +273,8 @@ github.com/gin-gonic/gin v1.7.0/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjX
|
|||
github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
|
||||
github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
|
||||
github.com/glycerine/vprint v0.0.0-20200730000117-76cea49a68ea h1:Uiuhuh77mImdrAMjPfw2V8tWw4AF6r9dxbNkECo23SA=
|
||||
github.com/glycerine/vprint v0.0.0-20200730000117-76cea49a68ea/go.mod h1:q7RHAiHHxYrXtGEkX14OuACg+cHODdKnVvTgpBnOzHk=
|
||||
github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-chi/chi v1.5.0/go.mod h1:REp24E+25iKvxgeTfHmdUoL5x15kBiDBlnIl5bCwe2k=
|
||||
|
|
@ -270,14 +288,17 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
|
|||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
|
||||
github.com/go-ldap/ldap/v3 v3.1.3/go.mod h1:3rbOH3jRS2u6jg2rJnKAMLE/xQyCKIveG2Sa/Cohzb8=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
|
||||
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
|
||||
github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
|
||||
github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
|
||||
github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
|
||||
github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
|
||||
|
|
@ -297,6 +318,7 @@ github.com/go-redis/redis/v7 v7.1.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRf
|
|||
github.com/go-redis/redis/v8 v8.0.0/go.mod h1:isLoQT/NFSP7V67lyvM9GmdvLdyZ7pEhsXvvyQtnQTo=
|
||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
|
|
@ -349,6 +371,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
|||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
|
|
@ -365,6 +388,8 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU
|
|||
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
|
||||
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
|
||||
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
|
|
@ -388,6 +413,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
|
|||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gomodule/redigo v1.7.0/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
|
|
@ -403,24 +429,34 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
|||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
|
||||
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20200615235658-03e1cf38a040/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20210423192551-a2663126120b h1:l2YRhr+YLzmSp7KJMswRVk/lO5SwoFIcCLzJsVj+YPc=
|
||||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20210423192551-a2663126120b/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
|
||||
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
|
|
@ -445,6 +481,7 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
|
|||
github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
|
||||
|
|
@ -521,6 +558,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
|
|||
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
|
||||
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
|
||||
github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ=
|
||||
|
|
@ -588,12 +626,16 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
|
|||
github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
|
||||
github.com/jaffee/commandeer v0.5.0 h1:241M9N+gHQmPyjIG+yy8GGcZPfzFuIyOmJHzm5ka92g=
|
||||
github.com/jaffee/commandeer v0.5.0/go.mod h1:kCwfuSvZ2T0NVEr3LDSo6fDUgi0xSBnAVDdkOKTtpLQ=
|
||||
github.com/jinzhu/gorm v1.9.1/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
|
||||
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
|
||||
|
|
@ -610,8 +652,9 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
|
|||
github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
|
|
@ -641,10 +684,14 @@ github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
|
|||
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
|
||||
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
|
||||
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
|
|
@ -669,10 +716,15 @@ github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
|
|||
github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
|
||||
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
|
||||
github.com/linkedin/goavro/v2 v2.11.1 h1:4cuAtbDfqkKnBXp9E+tRkIJGa6W6iAjwonwt8O1f4U0=
|
||||
github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
|
||||
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
|
||||
github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
|
|
@ -721,6 +773,7 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
|
|||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
|
|
@ -730,8 +783,9 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
|
||||
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
|
|
@ -789,6 +843,7 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T
|
|||
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
|
||||
github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
|
||||
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
|
||||
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
|
||||
github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
|
||||
|
|
@ -798,8 +853,12 @@ github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG
|
|||
github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
|
||||
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
|
||||
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pilosa/avro v0.0.0-20200626214113-bc1bf9fd41c1 h1:VEiqiLmebtgmPmpMhcByL5oXeLmDZyCLx+JmCJxDlG4=
|
||||
github.com/pilosa/avro v0.0.0-20200626214113-bc1bf9fd41c1/go.mod h1:evNEPnmTS6kdXAZXtqlZgYvM+FwoJ9YdGc7YbLb2RVU=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
|
|
@ -807,10 +866,13 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
|
|||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
|
||||
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
||||
|
|
@ -818,8 +880,11 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
|
|||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=
|
||||
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
|
||||
github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
|
||||
github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
|
|
@ -835,8 +900,10 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
|
|||
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
|
||||
github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
|
||||
github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE=
|
||||
github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
|
|
@ -845,16 +912,17 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
|
|||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=
|
||||
github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM=
|
||||
github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
|
||||
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/prom2json v1.3.1 h1:OogL5hsrJpLPz3jZ4LPz4sJRTtADzViCNRQoqrzUQvk=
|
||||
github.com/prometheus/prom2json v1.3.1/go.mod h1:A8Oy9aiQx4wrJY9ya1i4nHOySGmkVp5EO0aU1iSJR+g=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
|
||||
github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
|
|
@ -876,10 +944,11 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
|||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/segmentio/kafka-go v0.4.29 h1:4ujULpikzHG0HqKhjumDghFjy/0RRCSl/7lbriwQAH0=
|
||||
github.com/segmentio/kafka-go v0.4.29/go.mod h1:m1lXeqJtIFYZayv0shM/tjrAFljvWLTprxBHd+3PnaU=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg=
|
||||
github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
|
||||
github.com/shirou/gopsutil/v3 v3.22.5 h1:atX36I/IXgFiB81687vSiBI5zrMsxcIBkP9cQMJQoJA=
|
||||
github.com/shirou/gopsutil/v3 v3.22.5/go.mod h1:so9G9VzeHt/hsd0YwqprnjHnfARAUktauykSbr+y2gA=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
|
|
@ -903,25 +972,30 @@ github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJ
|
|||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
|
||||
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
|
||||
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
|
||||
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
||||
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
|
||||
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
|
||||
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
|
||||
github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw=
|
||||
github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
|
||||
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
|
||||
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
||||
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
|
||||
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
|
||||
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
|
||||
github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
|
||||
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
|
||||
|
|
@ -935,8 +1009,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
|||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
|
|
@ -957,10 +1032,10 @@ github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSR
|
|||
github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
|
||||
github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
|
||||
github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
|
||||
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
|
||||
github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ=
|
||||
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
|
||||
github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
|
||||
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
|
||||
github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
|
||||
github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
|
||||
|
|
@ -970,8 +1045,9 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
|
|||
github.com/twitchtv/twirp v8.1.1+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A=
|
||||
github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
|
||||
github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
|
||||
github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ=
|
||||
github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
|
||||
github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
|
||||
github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
|
|
@ -998,7 +1074,9 @@ github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV
|
|||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
|
||||
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
|
||||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
|
||||
|
|
@ -1012,14 +1090,17 @@ github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FB
|
|||
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
|
||||
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/blake3 v0.1.1 h1:Nbsts7DdKThRHHd+YNlqiGlRqGEF2bE2eXN+xQ1hsEs=
|
||||
github.com/zeebo/blake3 v0.1.1/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4=
|
||||
github.com/zeebo/pcg v1.0.0 h1:dt+dx+HvX8g7Un32rY9XWoYnd0NmKmrIzpHF7qiTDj0=
|
||||
github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
|
||||
github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
github.com/zenazn/goji v1.0.1/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
|
|
@ -1029,10 +1110,13 @@ go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
|
|||
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
|
||||
go.etcd.io/etcd v3.3.27+incompatible h1:5hMrpf6REqTHV2LW2OclNpRtxI0k9ZplMemJsMSWju0=
|
||||
go.etcd.io/etcd v3.3.27+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
|
||||
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
|
||||
go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc=
|
||||
go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
|
||||
go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
|
||||
go.etcd.io/etcd/client/v2 v2.305.4 h1:Dcx3/MYyfKcPNLpR4VVQUP5KgYrBeJtktBwEKkw08Ao=
|
||||
go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU=
|
||||
go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4=
|
||||
|
|
@ -1051,6 +1135,8 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
|||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
|
||||
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
|
||||
go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0=
|
||||
go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE=
|
||||
|
|
@ -1156,6 +1242,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
|
|||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
|
||||
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
|
|
@ -1167,6 +1254,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
|
|||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
|
||||
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
|
||||
|
|
@ -1186,9 +1275,11 @@ golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn
|
|||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
|
|
@ -1209,26 +1300,46 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
|
|||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
|
||||
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
|
||||
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g=
|
||||
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
|
@ -1237,10 +1348,12 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
@ -1300,14 +1413,22 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -1317,11 +1438,12 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
|
|
@ -1405,9 +1527,19 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY
|
|||
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
|
||||
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA=
|
||||
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
|
|
@ -1437,15 +1569,24 @@ google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/
|
|||
google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
|
||||
google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
|
||||
google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
|
||||
google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
|
||||
google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
|
||||
google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
|
|
@ -1475,8 +1616,22 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG
|
|||
google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20200528110217-3d3490e7e671/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
|
||||
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
|
||||
google.golang.org/genproto v0.0.0-20200726014623-da3ae01ef02d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
|
||||
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0=
|
||||
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
|
||||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
|
|
@ -1496,9 +1651,16 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
|
|||
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
|
||||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
|
||||
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
|
||||
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
|
||||
|
|
@ -1514,11 +1676,14 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
|
|||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
|
||||
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/DataDog/dd-trace-go.v1 v1.38.1 h1:nAKgcpJLXRHF56cKCP3bN8gTTQmmNAZFEblbyGKhKTo=
|
||||
gopkg.in/DataDog/dd-trace-go.v1 v1.38.1/go.mod h1:GBhK4yaMJ1h329ivtKAqRNe1EZ944UnZwtz5lh7CnJc=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 h1:PGIdqvwfpMUyUP+QAlAnKTSWQ671SmYjoou2/5j7HXk=
|
||||
gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
|
@ -1531,8 +1696,9 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
|
|||
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho=
|
||||
gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
|
||||
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/jinzhu/gorm.v1 v1.9.1/go.mod h1:56JJPUzbikvTVnoyP1nppSkbJ2L8sunqTBDY2fDrmFg=
|
||||
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
|
||||
|
|
|
|||
4
idk/.cloud-env.template
Normal file
4
idk/.cloud-env.template
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
PROFILE = <replace with local AWS profile name>
|
||||
AWS_REGION = <replace with region where image should live>
|
||||
ECR_URL = <replace with ECR URL for the account>
|
||||
ECR_REPO = <replace with ECR repo name>
|
||||
38
idk/Dockerfile
Normal file
38
idk/Dockerfile
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
ARG GO_VERSION=1.17.8
|
||||
|
||||
FROM golang:1.17.8-stretch as builder
|
||||
|
||||
WORKDIR /
|
||||
RUN ["apt-get","update","-y"]
|
||||
RUN ["apt-get","install","-y","git","unixodbc","unixodbc-dev","netcat", "build-essential","musl-tools"]
|
||||
RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
|
||||
WORKDIR /librdkafka
|
||||
RUN ["./configure", "--prefix", "/usr"]
|
||||
RUN ["make"]
|
||||
RUN ["make", "install"]
|
||||
|
||||
WORKDIR /featurebase
|
||||
|
||||
COPY . .
|
||||
|
||||
ARG MAKE_FLAGS
|
||||
ARG GO_BUILD_FLAGS
|
||||
|
||||
WORKDIR /featurebase/idk/
|
||||
|
||||
RUN make build GO_BUILD_FLAGS="-mod=vendor ${GO_BUILD_FLAGS}" ${MAKE_FLAGS}
|
||||
|
||||
FROM ubuntu:20.04 as runner
|
||||
|
||||
RUN ["apt-get","update","-y"]
|
||||
RUN ["apt-get","install","-y","unixodbc-dev","netcat","ca-certificates", "musl-tools"]
|
||||
|
||||
COPY --from=builder /idk/bin/* /usr/local/bin/
|
||||
|
||||
COPY /docker-sasl/ssl_keys /etc/kafka/secrets
|
||||
|
||||
# Verify that the linker can find everything.
|
||||
FROM runner as linkcheck
|
||||
RUN if [ -e /usr/local/bin/molecula-consumer-sql-odbc ] ; then ldd /usr/local/bin/molecula-consumer-sql-odbc; fi
|
||||
|
||||
FROM runner
|
||||
7
idk/Dockerfile-fakeIDP
Normal file
7
idk/Dockerfile-fakeIDP
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
FROM golang:1.17.8
|
||||
|
||||
WORKDIR /
|
||||
COPY fakeidp ./
|
||||
RUN go build .
|
||||
|
||||
ENTRYPOINT ["/fakeidp"]
|
||||
24
idk/Dockerfile-test
Normal file
24
idk/Dockerfile-test
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
ARG GO_VERSION=1.17.8
|
||||
|
||||
FROM golang:${GO_VERSION} as build_base
|
||||
|
||||
WORKDIR /
|
||||
RUN ["apt-get","update","-y"]
|
||||
RUN ["apt-get","install","-y","git","unixodbc","unixodbc-dev","netcat", "build-essential","musl-tools"]
|
||||
RUN ["git", "clone", "https://github.com/edenhill/librdkafka.git"]
|
||||
WORKDIR /librdkafka
|
||||
# need to install dependencies because librdkafka does not find openssl header otherwise
|
||||
RUN ["./configure", "--install-deps"]
|
||||
RUN ["./configure", "--prefix", "/usr"]
|
||||
RUN ["make"]
|
||||
RUN ["make", "install"]
|
||||
|
||||
FROM build_base
|
||||
|
||||
WORKDIR /go/src/github.com/molecula/featurebase/
|
||||
|
||||
COPY . .
|
||||
|
||||
WORKDIR /go/src/github.com/molecula/featurebase/idk/
|
||||
|
||||
CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."]
|
||||
8
idk/Dockerfile-wait
Normal file
8
idk/Dockerfile-wait
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
FROM ubuntu:18.04
|
||||
|
||||
RUN ["apt-get", "update", "-y"]
|
||||
RUN ["apt-get", "install", "-y", "postgresql-client", "curl", "netcat"]
|
||||
|
||||
ADD wait.sh /wait
|
||||
|
||||
ENTRYPOINT ["/wait"]
|
||||
311
idk/Makefile
Normal file
311
idk/Makefile
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
GO ?= go
|
||||
GOOS ?= $(shell $(GO) env GOOS)
|
||||
GOARCH ?= $(shell $(GO) env GOARCH)
|
||||
GO_VERSION=1.17.8
|
||||
GO_BUILD_FLAGS=
|
||||
ODBC_ENABLED=
|
||||
BINOUT ?= bin
|
||||
VERSION := $(shell git describe --tags 2> /dev/null || git rev-parse --verify --short=7 HEAD)
|
||||
VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
|
||||
BUILD_TIME := $(shell date -u +%FT%T%z)
|
||||
BRANCH_NAME ?= ""
|
||||
|
||||
# We allow setting a custom docker-compose "project". Multiple of the
|
||||
# same docker-compose environment can exist simultaneously as long as
|
||||
# they use different projects (the project name is prepended to
|
||||
# container names and such). This is useful in a CI environment where
|
||||
# we might be running multiple instances of the tests concurrently.
|
||||
PROJECT ?= idk
|
||||
DOCKER_COMPOSE = docker-compose -p $(PROJECT)
|
||||
|
||||
UNAME_P := $(shell uname -p)
|
||||
ifneq ($(filter arm%,$(UNAME_P)),)
|
||||
IS_ARM := 1
|
||||
else
|
||||
IS_ARM := 0
|
||||
endif
|
||||
|
||||
ifneq ($(filter Darwin%,$(shell uname)),)
|
||||
IS_MAC := 1
|
||||
else
|
||||
IS_MAC := 0
|
||||
endif
|
||||
|
||||
LDFLAGS="-X github.com/molecula/featurebase/idk.Version=$(VERSION) -X github.com/molecula/featurebase/idk.BuildTime=$(BUILD_TIME)"
|
||||
LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X github.com/molecula/featurebase/idk.Version=$(VERSION) -X github.com/molecula/featurebase/idk.BuildTime=$(BUILD_TIME) "
|
||||
export GOPRIVATE=github.com/molecula
|
||||
export CGO_ENABLED=1
|
||||
|
||||
# Cloud Deployment environment (dev, prod, staging, etc.), specified with the value of MCLOUD_ENV
|
||||
# E.g. `MCLOUD_ENV=test make docker-push-ecr`
|
||||
MCLOUD_ENV ?= dev
|
||||
MCLOUD_ENV_FILE=.cloud-env.$(MCLOUD_ENV)
|
||||
-include $(MCLOUD_ENV_FILE)
|
||||
|
||||
PROFILE ?=
|
||||
AWS_REGION ?=
|
||||
ECR_URL ?=
|
||||
ECR_REPO ?= idk
|
||||
|
||||
|
||||
BINS := $(patsubst cmd/%,$(BINOUT)/%,$(wildcard cmd/*))
|
||||
.PHONY: install build release-build release test $(BINS)
|
||||
|
||||
install:
|
||||
ifeq ($(filter $(IS_ARM),$(IS_MAC)), 1)
|
||||
export HOMEBREW_NO_AUTO_UPDATE=1 && [ "${PKG_CONFIG_PATH}" != "$$(brew --prefix openssl)/lib/pkgconfig" ] && (brew list pkg-config || brew install pkg-config) && (brew list librdkafka || brew install librdkafka) && (brew list openssl || brew install openssl) && export PKG_CONFIG_PATH="$$(brew --prefix openssl)/lib/pkgconfig"; \
|
||||
$(GO) install -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -tags dynamic ./cmd/...
|
||||
else
|
||||
$(GO) install -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) ./cmd/...
|
||||
endif
|
||||
|
||||
$(BINS):
|
||||
$(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o $@ ./cmd/$(notdir $@)
|
||||
|
||||
$(BINOUT)/molecula-consumer-sql-odbc:
|
||||
CGO_ENABLED=1 $(GO) build -tags=odbc -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o $@ ./cmd/molecula-consumer-sql
|
||||
|
||||
build_non_cgo:
|
||||
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/ingester ./cmd/ingester
|
||||
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-csv ./cmd/molecula-consumer-csv
|
||||
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-static ./cmd/molecula-consumer-kafka-static
|
||||
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-sql ./cmd/molecula-consumer-sql
|
||||
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-github ./cmd/molecula-consumer-github
|
||||
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kinesis ./cmd/molecula-consumer-kinesis
|
||||
|
||||
build_cgo:
|
||||
ifeq ($(GOARCH), arm64)
|
||||
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka ./cmd/molecula-consumer-kafka
|
||||
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-delete ./cmd/molecula-consumer-kafka-delete
|
||||
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/kafkagen ./cmd/kafkagen
|
||||
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/kafkaput ./cmd/kafkaput
|
||||
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/datagen ./cmd/datagen
|
||||
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/bankgen ./cmd/bankgen
|
||||
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-sasl ./cmd/molecula-consumer-kafka-sasl
|
||||
endif
|
||||
ifeq ($(GOARCH), amd64)
|
||||
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka ./cmd/molecula-consumer-kafka
|
||||
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-delete ./cmd/molecula-consumer-kafka-delete
|
||||
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/kafkagen ./cmd/kafkagen
|
||||
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/kafkaput ./cmd/kafkaput
|
||||
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/datagen ./cmd/datagen
|
||||
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/bankgen ./cmd/bankgen
|
||||
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-sasl ./cmd/molecula-consumer-kafka-sasl
|
||||
endif
|
||||
|
||||
CGO_ENABLED=1 $(GO) build -tags=odbc -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-sql-odbc ./cmd/molecula-consumer-sql
|
||||
|
||||
BUILD_CGO ?= 0
|
||||
build:
|
||||
@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
|
||||
make build_non_cgo
|
||||
ifeq ($(BUILD_CGO), 1)
|
||||
make build_cgo
|
||||
endif
|
||||
|
||||
release-build:
|
||||
mkdir -p build/molecula-consumer-$(VERSION_ID)/
|
||||
$(MAKE) build BINOUT=build/molecula-consumer-$(VERSION_ID)/
|
||||
tar -cvz -C build -f build/molecula-consumer-$(VERSION_ID).tar.gz molecula-consumer-$(VERSION_ID)/
|
||||
@echo Created release build: build/molecula-consumer-$(VERSION_ID).tar.gz
|
||||
|
||||
release: check-clean
|
||||
make release-build GOOS=linux GOARCH=amd64
|
||||
make release-build GOOS=linux GOARCH=arm64
|
||||
make release-build GOOS=darwin GOARCH=amd64
|
||||
make release-build GOOS=darwin GOARCH=arm64
|
||||
|
||||
test:
|
||||
$(GO) test ./...
|
||||
|
||||
test-local: testenv
|
||||
IDK_TEST_CERT_PATH=$(shell pwd)/testenv/certs \
|
||||
IDK_TEST_KAFKA_HOST=localhost:9092 \
|
||||
IDK_TEST_PILOSA_GRPC_HOST=localhost:20101 \
|
||||
IDK_TEST_PILOSA_HOST=localhost:10101 \
|
||||
IDK_TEST_PILOSA_TLS_HOST=https://localhost:10111 \
|
||||
IDK_TEST_REGISTRY_HOST=localhost:8081 \
|
||||
$(GO) test $(TESTFLAGS) ./...
|
||||
|
||||
vendor: ../go.mod
|
||||
$(GO) mod vendor
|
||||
|
||||
build-%:
|
||||
$(DOCKER_COMPOSE) build $*
|
||||
|
||||
pull-%:
|
||||
$(DOCKER_COMPOSE) pull $*
|
||||
|
||||
save-%-logs:
|
||||
$(DOCKER_COMPOSE) logs $* > ./testdata/$(PROJECT)_$*_logs.txt
|
||||
|
||||
|
||||
start-all: .pulled testenv build-wait
|
||||
echo "branch name" ${BRANCH_NAME}
|
||||
$(DOCKER_COMPOSE) up -d zookeeper
|
||||
$(DOCKER_COMPOSE) run -T wait zookeeper 'echo "ruok" | nc -w 2 zookeeper 2181 | grep imok'
|
||||
$(DOCKER_COMPOSE) up -d kafka
|
||||
$(DOCKER_COMPOSE) up -d schema-registry
|
||||
$(DOCKER_COMPOSE) up -d postgres
|
||||
$(DOCKER_COMPOSE) run -T wait postgres pg_isready -h postgres -p 5432 -U postgres
|
||||
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa
|
||||
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa-tls
|
||||
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa-auth
|
||||
$(DOCKER_COMPOSE) run -T wait pilosa curl --silent --fail http://pilosa:10101/status
|
||||
$(DOCKER_COMPOSE) run -T wait pilosa-tls curl --silent --cacert /certs/ca.crt --key /certs/theclient.key --cert /certs/theclient.crt --fail https://pilosa-tls:10111/status
|
||||
$(DOCKER_COMPOSE) run -T wait pilosa-auth curl --silent --fail http://pilosa-auth:10105/version
|
||||
$(DOCKER_COMPOSE) run -T wait kafka nc -z kafka 9092
|
||||
$(DOCKER_COMPOSE) run -T wait schema-registry curl --silent --fail http://schema-registry:8081/config
|
||||
|
||||
start-postgres: build-wait testenv .pulled
|
||||
$(DOCKER_COMPOSE) up -d postgres
|
||||
$(DOCKER_COMPOSE) run -T wait postgres pg_isready -h postgres -p 5432 -U postgres
|
||||
|
||||
start-pilosa: build-pilosa start-postgres build-wait testenv .pulled
|
||||
$(DOCKER_COMPOSE) up -d pilosa
|
||||
$(DOCKER_COMPOSE) run -T wait pilosa curl --silent --fail http://pilosa:10101/status
|
||||
|
||||
start-pilosa-tls: build-pilosa-tls build-wait testenv .pulled
|
||||
$(DOCKER_COMPOSE) up -d pilosa-tls
|
||||
$(DOCKER_COMPOSE) run -T wait pilosa-tls curl --silent --cacert /certs/ca.crt --key /certs/theclient.key --cert /certs/theclient.crt --fail https://pilosa-tls:10111/status
|
||||
|
||||
start-pilosa-auth: build-pilosa-auth build-wait testenv .pulled
|
||||
$(DOCKER_COMPOSE) up -d pilosa-auth
|
||||
$(DOCKER_COMPOSE) run -T wait pilosa-auth curl --silent --fail http://pilosa-auth:10105/version
|
||||
|
||||
start-zookeeper: pull-zookeeper testenv
|
||||
$(DOCKER_COMPOSE) up -d zookeeper
|
||||
$(DOCKER_COMPOSE) run -T wait zookeeper 'echo "ruok" | nc -w 2 zookeeper 2181 | grep imok'
|
||||
|
||||
start-kafka: pull-kafka start-zookeeper build-wait testenv
|
||||
$(DOCKER_COMPOSE) up -d kafka
|
||||
$(DOCKER_COMPOSE) run -T wait kafka nc -z kafka 9092
|
||||
|
||||
start-schema-registry: start-zookeeper start-kafka pull-schema-registry build-wait testenv
|
||||
$(DOCKER_COMPOSE) up -d schema-registry
|
||||
$(DOCKER_COMPOSE) run -T wait schema-registry curl --silent --fail http://schema-registry:8081/config
|
||||
|
||||
startup-old: start-postgres start-pilosa start-pilosa-tls start-pilosa-auth start-zookeeper start-kafka start-schema-registry
|
||||
startup: start-all
|
||||
|
||||
shutdown:
|
||||
$(DOCKER_COMPOSE) down -v --remove-orphans
|
||||
rm .pulled
|
||||
|
||||
test-all: testenv
|
||||
$(MAKE) startup
|
||||
$(MAKE) test-run
|
||||
$(DOCKER_COMPOSE) logs kafka | grep "Kafka version"
|
||||
$(MAKE) shutdown
|
||||
|
||||
|
||||
test-all-race: testenv
|
||||
$(MAKE) startup
|
||||
$(MAKE) test-run-race
|
||||
$(MAKE) shutdown
|
||||
|
||||
test-all-kafka-sasl: testenv
|
||||
$(MAKE) startup
|
||||
$(MAKE) test-run-kafka-sasl
|
||||
$(MAKE) shutdown
|
||||
|
||||
TCMD ?= ./...
|
||||
# do "make startup", then e.g. "make test-run-local TCMD='-run=MyFavTest ./kafka'"
|
||||
test-run-local:
|
||||
pwd
|
||||
$(DOCKER_COMPOSE) build idk-test
|
||||
$(DOCKER_COMPOSE) run -T idk-test go test -mod=vendor -tags=odbc,dynamic $(TCMD)
|
||||
|
||||
|
||||
TPKG ?= ./...
|
||||
test-run: testenv vendor
|
||||
$(DOCKER_COMPOSE) build idk-test
|
||||
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -json -coverprofile=/testdata/$(PROJECT)_coverage.out | tee /testdata/$(PROJECT)_report.out"
|
||||
$(DOCKER_COMPOSE) run -T idk-test /go/src/github.com/molecula/featurebase/idk/reingest_test.sh
|
||||
|
||||
|
||||
test-run-race: testenv vendor
|
||||
$(DOCKER_COMPOSE) build idk-test
|
||||
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -race -covermode=atomic -tags=dynamic $(TPKG) -coverpkg=$(TPKG) -timeout=30m -json -coverprofile=/testdata/$(PROJECT)_coverage.out | tee /testdata/$(PROJECT)_report.out"
|
||||
|
||||
test-run-kafka-sasl: testenv vendor
|
||||
$(DOCKER_COMPOSE) build idk-test
|
||||
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v --tags=kafka_sasl -mod=vendor -race -timeout=30m $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -json -coverprofile=/testdata/$(PROJECT)_coverage.out | tee /testdata/$(PROJECT)_report.out"
|
||||
|
||||
.pulled:
|
||||
$(DOCKER_COMPOSE) pull
|
||||
touch .pulled
|
||||
|
||||
testenv: testenv/certs
|
||||
|
||||
testenv/certs:
|
||||
certstrap --depot-path testenv/certs init --common-name ca --passphrase "" --expires "100 years"
|
||||
certstrap --depot-path=testenv/certs request-cert --domain localhost --common-name localhost --passphrase ""
|
||||
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" localhost
|
||||
certstrap --depot-path=testenv/certs request-cert --domain pilosa-tls --common-name pilosa-tls --passphrase ""
|
||||
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" pilosa-tls
|
||||
certstrap --depot-path=testenv/certs request-cert --domain theclient --common-name theclient --passphrase ""
|
||||
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" theclient
|
||||
|
||||
# Create release using Docker
|
||||
docker-release:
|
||||
@if [ $(IS_ARM) = 1 ]; then \
|
||||
echo Building linux-amd64 image on $(UNAME_P) is not supported; \
|
||||
else \
|
||||
$(MAKE) docker-build GOOS=linux GOARCH=amd64; \
|
||||
fi
|
||||
$(MAKE) docker-build GOOS=darwin GOARCH=amd64
|
||||
$(MAKE) docker-build GOOS=darwin GOARCH=arm64
|
||||
$(MAKE) docker-build GOOS=linux GOARCH=arm64
|
||||
|
||||
|
||||
# This allows multiple concurrent builds to happen in CI without
|
||||
# creating container name conflicts and such. (different BUILD_NAMEs
|
||||
# are passed in from gitlab-ci.yml)
|
||||
BUILD_NAME ?= idk-build
|
||||
|
||||
# Build a release in Docker
|
||||
docker-build: vendor
|
||||
DOCKER_BUILDKIT=0 docker build \
|
||||
-f ../idk/Dockerfile \
|
||||
--build-arg GO_VERSION=$(GO_VERSION) \
|
||||
--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
|
||||
--build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
|
||||
--target builder \
|
||||
--tag idk:$(BUILD_NAME) ../.
|
||||
mkdir -p build/idk-$(GOOS)-$(GOARCH)
|
||||
docker create --name $(BUILD_NAME) idk:$(BUILD_NAME)
|
||||
docker cp $(BUILD_NAME):/featurebase/idk/bin/. ./build/idk-$(GOOS)-$(GOARCH)
|
||||
docker rm $(BUILD_NAME)
|
||||
|
||||
# Create Docker image from Dockerfile
|
||||
docker-image: vendor
|
||||
docker build \
|
||||
--build-arg GO_VERSION=$(GO_VERSION) \
|
||||
--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
|
||||
--tag registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) .
|
||||
@echo Created docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
|
||||
|
||||
docker: docker-image
|
||||
|
||||
docker-tag-push:
|
||||
docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
|
||||
@echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
|
||||
|
||||
clean:
|
||||
$(MAKE) shutdown
|
||||
rm -rf testenv
|
||||
|
||||
# Error out if there are untracked changes in Git
|
||||
check-clean:
|
||||
ifndef SKIP_CHECK_CLEAN
|
||||
$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
|
||||
endif
|
||||
|
||||
docker-push-ecr: docker-image aws-login
|
||||
docker tag idk:$(VERSION) $(ECR_URL)/$(ECR_REPO):$(VERSION)
|
||||
aws ecr get-login-password --region $(AWS_REGION) --profile $(PROFILE)| docker login --username AWS --password-stdin $(ECR_URL)
|
||||
docker push $(ECR_URL)/$(ECR_REPO):$(VERSION)
|
||||
|
||||
aws-login:
|
||||
aws sso login --profile $(PROFILE)
|
||||
124
idk/README.md
Normal file
124
idk/README.md
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
# idk: Ingest Development Kit
|
||||
|
||||
## Integration tests
|
||||
|
||||
To run the tests, you will need to install the following dependencies:
|
||||
|
||||
1. [Docker](https://docs.docker.com/install/)
|
||||
2. [Docker Compose](https://docs.docker.com/compose/install/)
|
||||
3. [Certstrap](https://github.com/square/certstrap)
|
||||
|
||||
In addition to these dependancies, you will need to be added to the moleculacorp [Dockerhub](https://hub.docker.com/orgs/moleculacorp) account.
|
||||
|
||||
First start the test environment. This is a docker-compose environment that includes pilosa and a confluent kafka stack. Run the following to start those services:
|
||||
|
||||
make startup
|
||||
|
||||
To build and run the integration tests, run:
|
||||
|
||||
make test-run
|
||||
|
||||
Then to shut down the test environment, run:
|
||||
|
||||
make shutdown
|
||||
|
||||
You can run all of the previous commands by calling test-all:
|
||||
|
||||
make test-all
|
||||
|
||||
The previous command is equivalent to running the following:
|
||||
|
||||
make startup
|
||||
sleep 30 # wait for services to come up
|
||||
make test-run
|
||||
make shutdown
|
||||
|
||||
To run an individual test, you can run the command directly using docker-compose. Note that you must run `docker-compose build idk-test` for docker to run the latest code. Modify the following as needed:
|
||||
|
||||
make startup
|
||||
docker-compose build idk-test
|
||||
docker-compose run idk-test /usr/local/go/bin/go test -count=1 -mod=vendor -run=TestCmdMainOne ./kafka
|
||||
|
||||
To shutdown and reset the environment:
|
||||
|
||||
make clean
|
||||
|
||||
## Running dependencies locally (rather than in docker) "make test-local"
|
||||
|
||||
This is for running the tests locally and *not* in Docker... so you
|
||||
have to be running a bunch of stuff natively on your machine.
|
||||
|
||||
Run Pilosa with default config:
|
||||
|
||||
pilosa server
|
||||
|
||||
Run another pilosa like
|
||||
|
||||
pilosa server --config=pilosa-sec-test.conf
|
||||
|
||||
which will run Pilosa with TLS using certs in testenv. (`make testenv` first if you haven't).
|
||||
|
||||
You also need to be running the Confluent stack which, after you've
|
||||
installed it from Confluent's site might look something like:
|
||||
|
||||
export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_66.jdk/Contents/Home
|
||||
confluent local destroy && confluent local start schema-registry
|
||||
|
||||
Or it might not! You may not need the first line, but if you have the
|
||||
wrong Java version by default, that's how you set it. The second line
|
||||
may change depending on what version of the confluent stack you get. According to
|
||||
|
||||
confluent version
|
||||
|
||||
I'm running:
|
||||
|
||||
Version: v0.212.0
|
||||
Git Ref: 2b04985
|
||||
|
||||
Use the test-local make target:
|
||||
|
||||
make test-local
|
||||
|
||||
This sets a number of environment variables (which it prints when you
|
||||
run it), and should set them correctly if you follow the instructions
|
||||
above, but if you're running things on non-default ports you may need
|
||||
to tweak them.
|
||||
|
||||
## CSV Ingester
|
||||
|
||||
1. make sure you're running Pilosa (localhost:10101 for these instructions)
|
||||
|
||||
molecula-consumer-csv --primary-key-fields=asset_tag -i sample-index --files sample.csv
|
||||
|
||||
asset_tag__String,fan_time__RecordTime_2006-01-02,fan_val__String_F_YMD
|
||||
ABCD,2019-01-02,70%
|
||||
ABCD,2019-01-03,20%
|
||||
ABCD,2019-01-04,30%
|
||||
BEDF,2019-01-02,70%
|
||||
BEDF,2019-01-05,90%
|
||||
BEDF,2019-01-08,10%
|
||||
BEDF,2019-01-08,20%
|
||||
ABCD,2019-01-30,40%
|
||||
|
||||
## Datagen
|
||||
Datagen is an internal command-line tool to generate various application-specific datasets, and ingest them directly into Pilosa. After running `make install`, run `datagen` with no arguments to see a list of available "sources".
|
||||
|
||||
When running Datagen with local Docker stacks, make sure to add individual docker stacks to the `/etc/hosts` file:
|
||||
|
||||
127.0.0.1 kafka
|
||||
127.0.0.1 pilosa
|
||||
127.0.0.1 <docker stack>
|
||||
|
||||
## ODBC Support
|
||||
|
||||
By default, the SQL ingester is not built with ODBC support.
|
||||
This is because it uses CGO with extra dependencies, and the resulting binaries are not portable.
|
||||
In order to build with ODBC support, it is necessary to install `unixODBC` to the system.
|
||||
Then run:
|
||||
```
|
||||
make bin/molecula-consumer-sql-odbc
|
||||
```
|
||||
|
||||
Different Linux distros will store certain libraries in different locations.
|
||||
ODBC uses dynamic library loading to handle drivers, so full static linking of dependencies is not possible.
|
||||
It is therefore necessary to build on a system with the same distro and the same software versions as the target machine.
|
||||
84
idk/api/codec.go
Normal file
84
idk/api/codec.go
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// newJSONCodec creates a codec to stream a JSON array of objects, each of which represents a record.
|
||||
// This returns an error if the data stream does not contain the start of a valid JSON array.
|
||||
func newJSONCodec(r io.Reader) (*jsonCodec, error) {
|
||||
// Create a JSON decoder.
|
||||
d := json.NewDecoder(r)
|
||||
|
||||
// Enable the json.Number type in parsed output.
|
||||
// This allows us to retain precision on numbers, rather than converting them to floating point.
|
||||
d.UseNumber()
|
||||
|
||||
// Do not allow unknown fields.
|
||||
d.DisallowUnknownFields()
|
||||
|
||||
// Read the opening bracket of the array.
|
||||
token, err := d.Token()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
// The input included nothing (except maybe whitespace).
|
||||
// This is not expected.
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil, errors.Wrap(err, "parsing start of JSON data")
|
||||
}
|
||||
switch token {
|
||||
case json.Delim('['):
|
||||
// This is the start of the array of records.
|
||||
|
||||
case json.Delim('{'):
|
||||
// The input is a JSON object instead of an array.
|
||||
// This seems like an easy user error, so provide a descriptive error.
|
||||
return nil, errors.New("expected an array of records but found an object")
|
||||
|
||||
default:
|
||||
// This is a number or something.
|
||||
return nil, errors.Errorf("expected array of records but got %v", token)
|
||||
}
|
||||
|
||||
return &jsonCodec{d: d}, nil
|
||||
}
|
||||
|
||||
type record struct {
|
||||
PrimaryKey interface{} `json:"primary-key"`
|
||||
Values map[string]interface{} `json:"values"`
|
||||
}
|
||||
|
||||
type jsonCodec struct {
|
||||
d *json.Decoder
|
||||
}
|
||||
|
||||
func (c *jsonCodec) Next() (record, error) {
|
||||
if !c.d.More() {
|
||||
// There are no more records in the stream.
|
||||
// Read the ']'.
|
||||
_, err := c.d.Token()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
// If the closing bracket is missing, then the stream was terminated prematurely.
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return record{}, errors.Wrap(err, "parsing end of record array")
|
||||
}
|
||||
|
||||
// There are no more records.
|
||||
return record{}, io.EOF
|
||||
}
|
||||
|
||||
// Parse the record.
|
||||
var r record
|
||||
err := c.d.Decode(&r)
|
||||
if err != nil {
|
||||
return record{}, errors.Wrap(err, "parsing record")
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
112
idk/api/codec_test.go
Normal file
112
idk/api/codec_test.go
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestJSONCodec(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Ensure that an empty input is handled gracefully.
|
||||
t.Run("MissingInput", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := newJSONCodec(strings.NewReader(``))
|
||||
assert.EqualError(t, err, "parsing start of JSON data: unexpected EOF")
|
||||
})
|
||||
|
||||
// Ensure that something other than an array is handled gracefully.
|
||||
t.Run("Object", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := newJSONCodec(strings.NewReader(`{}`))
|
||||
assert.EqualError(t, err, "expected an array of records but found an object")
|
||||
})
|
||||
|
||||
// Ensure that an incomplete array is handled gracefully.
|
||||
t.Run("IncompleteArray", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
codec, err := newJSONCodec(strings.NewReader(`[`))
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
|
||||
_, err = codec.Next()
|
||||
assert.EqualError(t, err, "parsing end of record array: unexpected EOF")
|
||||
})
|
||||
|
||||
// Test normal behavior.
|
||||
t.Run("Normal", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
expect := []record{
|
||||
{
|
||||
PrimaryKey: json.Number("2"),
|
||||
Values: map[string]interface{}{
|
||||
"int": json.Number("1"),
|
||||
"decimal": json.Number("2.01"),
|
||||
"string": "h",
|
||||
"idset": []interface{}{
|
||||
json.Number("1"),
|
||||
json.Number("2"),
|
||||
json.Number("4"),
|
||||
},
|
||||
"stringset": []interface{}{"plugh"},
|
||||
},
|
||||
},
|
||||
{
|
||||
PrimaryKey: "h",
|
||||
Values: map[string]interface{}{},
|
||||
},
|
||||
}
|
||||
codec, err := newJSONCodec(strings.NewReader(`[
|
||||
{
|
||||
"primary-key": 2,
|
||||
"values": {
|
||||
"int": 1,
|
||||
"decimal": 2.01,
|
||||
"string": "h",
|
||||
"idset": [1, 2, 4],
|
||||
"stringset": ["plugh"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"primary-key": "h",
|
||||
"values": {}
|
||||
}
|
||||
]`))
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
|
||||
out, err := decodeAll(codec)
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
|
||||
assert.Equal(t, expect, out)
|
||||
})
|
||||
}
|
||||
|
||||
func decodeAll(c *jsonCodec) ([]record, error) {
|
||||
var out []record
|
||||
for {
|
||||
rec, err := c.Next()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out = append(out, rec)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
183
idk/api/schema.go
Normal file
183
idk/api/schema.go
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
pilosa "github.com/molecula/featurebase/v3"
|
||||
pilosaclient "github.com/molecula/featurebase/v3/client"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type schema struct {
|
||||
IndexName string `json:"index-name"`
|
||||
IfNotExists bool `json:"if-not-exists,omitempty"`
|
||||
PrimaryKeyType string `json:"primary-key-type"`
|
||||
|
||||
Fields []schemaField `json:"fields"`
|
||||
}
|
||||
|
||||
type schemaField struct {
|
||||
FieldName string `json:"field-name"`
|
||||
FieldType string `json:"field-type"`
|
||||
FieldOptions struct {
|
||||
TimeQuantum string `json:"time-quantum,omitempty"`
|
||||
CacheSize int `json:"cache-size,omitempty"`
|
||||
CacheType string `json:"cache-type,omitempty"`
|
||||
EnforceMutualExclusion bool `json:"enforce-mutual-exclusion,omitempty"`
|
||||
Scale int `json:"scale,omitempty"`
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
} `json:"field-options,omitempty"`
|
||||
}
|
||||
|
||||
// DecodeSchema takes JSON stream (io.Reader - mainly http request body)
|
||||
// and tries to decode it into pilosa.Schema.
|
||||
// The function returns also a map: index-name -> if-not-exists option,
|
||||
// so we can swallow "conflict" error in case of creating an index with
|
||||
// the name as existing one.
|
||||
func DecodeSchema(r io.Reader) (*pilosaclient.Schema, map[string]bool, error) {
|
||||
pilosaSchema := pilosaclient.NewSchema()
|
||||
|
||||
// map: index-name -> if-not-exists option
|
||||
ifNotExists := make(map[string]bool)
|
||||
|
||||
// Create a JSON decoder.
|
||||
d := json.NewDecoder(r)
|
||||
|
||||
// This is just a generic loop-approach where we can iterate over JSON objects
|
||||
// (stream of indexes), but on daily-bases we should just decode one index.
|
||||
for d.More() {
|
||||
var apiSchema schema
|
||||
if err := d.Decode(&apiSchema); err != nil {
|
||||
return nil, nil, errors.Wrap(err, "decoding schema")
|
||||
}
|
||||
|
||||
err := apiSchema.applyToPilosa(pilosaSchema)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrapf(err, "applying schema of index %q", apiSchema.IndexName)
|
||||
}
|
||||
|
||||
ifNotExists[apiSchema.IndexName] = apiSchema.IfNotExists
|
||||
}
|
||||
|
||||
return pilosaSchema, ifNotExists, nil
|
||||
}
|
||||
|
||||
// applyToPilosa validates an index schema and adds it to a Pilosa schema.
|
||||
func (s schema) applyToPilosa(ps *pilosaclient.Schema) error {
|
||||
if err := pilosa.ValidateName(s.IndexName); err != nil {
|
||||
return errors.Wrap(err, "validating index name")
|
||||
}
|
||||
|
||||
var keyed bool
|
||||
switch s.PrimaryKeyType {
|
||||
case "string":
|
||||
keyed = true
|
||||
case "uint":
|
||||
keyed = false
|
||||
case "auto":
|
||||
return errors.New("auto primary keys not yet implemented")
|
||||
default:
|
||||
return errors.Errorf(`invalid primary-key-type %q, it must be "string", "uint" or "auto"`, s.PrimaryKeyType)
|
||||
}
|
||||
|
||||
idx := ps.Index(s.IndexName,
|
||||
pilosaclient.OptIndexTrackExistence(true),
|
||||
pilosaclient.OptIndexKeys(keyed),
|
||||
)
|
||||
|
||||
for _, f := range s.Fields {
|
||||
err := f.applyToPilosa(idx)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "applying field %q from schema", f.FieldName)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyToPilosa validates a field schema and adds it to an index schema.
|
||||
func (f schemaField) applyToPilosa(idx *pilosaclient.Index) error {
|
||||
if err := pilosa.ValidateName(f.FieldName); err != nil {
|
||||
return errors.Wrap(err, "validating field name")
|
||||
}
|
||||
|
||||
if idx.HasField(f.FieldName) {
|
||||
return errors.New("field defined multiple times")
|
||||
}
|
||||
|
||||
var opts []pilosaclient.FieldOption
|
||||
switch f.FieldType {
|
||||
case "id":
|
||||
switch {
|
||||
case f.FieldOptions.EnforceMutualExclusion:
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeMutex(pilosaclient.CacheType(f.FieldOptions.CacheType), f.FieldOptions.CacheSize)}
|
||||
case f.FieldOptions.TimeQuantum != "":
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeTime(pilosaclient.TimeQuantum(f.FieldOptions.TimeQuantum))}
|
||||
if f.FieldOptions.TTL != "" {
|
||||
ttl, err := time.ParseDuration(f.FieldOptions.TTL)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to parse TTL from field %s", f.FieldName)
|
||||
} else {
|
||||
opts = append(opts, pilosaclient.OptFieldTTL(ttl))
|
||||
}
|
||||
}
|
||||
default:
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeSet(pilosaclient.CacheType(f.FieldOptions.CacheType), f.FieldOptions.CacheSize)}
|
||||
}
|
||||
|
||||
case "string":
|
||||
switch {
|
||||
case f.FieldOptions.EnforceMutualExclusion:
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeMutex(pilosaclient.CacheType(f.FieldOptions.CacheType), f.FieldOptions.CacheSize)}
|
||||
case f.FieldOptions.TimeQuantum != "":
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeTime(pilosaclient.TimeQuantum(f.FieldOptions.TimeQuantum))}
|
||||
if f.FieldOptions.TTL != "" {
|
||||
ttl, err := time.ParseDuration(f.FieldOptions.TTL)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "unable to parse TTL from field %s", f.FieldName)
|
||||
} else {
|
||||
opts = append(opts, pilosaclient.OptFieldTTL(ttl))
|
||||
}
|
||||
}
|
||||
default:
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeSet(pilosaclient.CacheType(f.FieldOptions.CacheType), f.FieldOptions.CacheSize)}
|
||||
}
|
||||
|
||||
case "bool":
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeBool()}
|
||||
|
||||
case "int":
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeInt()}
|
||||
|
||||
case "decimal":
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeDecimal(int64(f.FieldOptions.Scale))}
|
||||
|
||||
case "timestamp":
|
||||
epoch := time.Unix(0, 0)
|
||||
if f.FieldOptions.Epoch != "" {
|
||||
var err error
|
||||
epoch, err = time.Parse(time.RFC3339, f.FieldOptions.Epoch)
|
||||
if err != nil {
|
||||
return errors.Errorf("invalid epoch time %q for layout %q", f.FieldOptions.Epoch, time.RFC3339)
|
||||
}
|
||||
}
|
||||
|
||||
unit := f.FieldOptions.Unit
|
||||
if !pilosa.IsValidTimeUnit(unit) {
|
||||
return errors.Errorf("invalid time unit %q", f.FieldOptions.Unit)
|
||||
}
|
||||
|
||||
opts = []pilosaclient.FieldOption{pilosaclient.OptFieldTypeTimestamp(epoch, unit)}
|
||||
|
||||
default:
|
||||
return errors.Errorf(`invalid field-type %q, it must be "id", "string", "bool", "int", "decimal" or "timestamp"`, f.FieldType)
|
||||
}
|
||||
|
||||
idx.Field(f.FieldName, opts...)
|
||||
|
||||
return nil
|
||||
}
|
||||
139
idk/api/schema_test.go
Normal file
139
idk/api/schema_test.go
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
pilosaclient "github.com/molecula/featurebase/v3/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestApplyToPilosa(t *testing.T) {
|
||||
client, err := pilosaclient.NewClient("pilosa:10101")
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
err := client.Close()
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
schema, err := client.Schema()
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
if schema.HasIndex("apitestingest_unkeyed") {
|
||||
err := client.DeleteIndexByName("apitestingest_unkeyed")
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
schema, err = client.Schema()
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
|
||||
unkeyedIdx := schema.Index("apitestingest_unkeyed", pilosaclient.OptIndexTrackExistence(true))
|
||||
|
||||
err = client.SyncSchema(schema)
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
err := client.DeleteIndex(unkeyedIdx)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
type FieldOpt struct { //nolint:unused
|
||||
TimeQuantum string `json:"time-quantum,omitempty"`
|
||||
CacheSize int `json:"cache-size,omitempty"`
|
||||
CacheType string `json:"cache-type,omitempty"`
|
||||
EnforceMutualExclusion bool `json:"enforce-mutual-exclusion,omitempty"`
|
||||
Scale int `json:"scale,omitempty"`
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
schemaField schemaField
|
||||
expErr string
|
||||
expOption string
|
||||
}{
|
||||
{
|
||||
name: "id_ttl",
|
||||
schemaField: schemaField{
|
||||
FieldName: "id_ttl",
|
||||
FieldType: "id",
|
||||
FieldOptions: FieldOpt{"YMD", 0, "", false, 0, "", "", "11s"},
|
||||
},
|
||||
expErr: "",
|
||||
expOption: `"{"options":{"noStandardView":false,"timeQuantum":"YMD","ttl":"11s","type":"time"}}"}`,
|
||||
},
|
||||
{
|
||||
name: "id_bad_ttl",
|
||||
schemaField: schemaField{
|
||||
FieldName: "id_bad_ttl",
|
||||
FieldType: "id",
|
||||
FieldOptions: FieldOpt{"YMD", 0, "", false, 0, "", "", "bad-ttl"},
|
||||
},
|
||||
expErr: "unable to parse TTL",
|
||||
expOption: `"{"options":{"type":"set"}}"}`,
|
||||
},
|
||||
{
|
||||
name: "id_ttl_without_time_quantum",
|
||||
schemaField: schemaField{
|
||||
FieldName: "id_ttl_without_time_quantum",
|
||||
FieldType: "id",
|
||||
FieldOptions: FieldOpt{"", 0, "", false, 0, "", "", "12s"},
|
||||
},
|
||||
expErr: "",
|
||||
expOption: `options: "{"options":{"type":"set"}}"}`,
|
||||
},
|
||||
{
|
||||
name: "string_ttl",
|
||||
schemaField: schemaField{
|
||||
FieldName: "string_ttl",
|
||||
FieldType: "string",
|
||||
FieldOptions: FieldOpt{"YMD", 0, "", false, 0, "", "", "13s"},
|
||||
},
|
||||
expErr: "",
|
||||
expOption: `"{"options":{"noStandardView":false,"timeQuantum":"YMD","ttl":"13s","type":"time"}}"}`,
|
||||
},
|
||||
{
|
||||
name: "string_bad_ttl",
|
||||
schemaField: schemaField{
|
||||
FieldName: "string_bad_ttl",
|
||||
FieldType: "string",
|
||||
FieldOptions: FieldOpt{"YMD", 0, "", false, 0, "", "", "bad-ttl"},
|
||||
},
|
||||
expErr: "unable to parse TTL",
|
||||
expOption: `"{"options":{"type":"set"}}"}`,
|
||||
},
|
||||
{
|
||||
name: "string_ttl_without_time_quantum",
|
||||
schemaField: schemaField{
|
||||
FieldName: "string_ttl_without_time_quantum",
|
||||
FieldType: "string",
|
||||
FieldOptions: FieldOpt{"", 0, "", false, 0, "", "", "14s"},
|
||||
},
|
||||
expErr: "",
|
||||
expOption: `options: "{"options":{"type":"set"}}"}`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := (test.schemaField).applyToPilosa(unkeyedIdx)
|
||||
if err != nil && !strings.Contains(err.Error(), test.expErr) {
|
||||
t.Errorf("expected error: '%s', got: '%s'", test.expErr, err.Error())
|
||||
}
|
||||
|
||||
if !strings.Contains(unkeyedIdx.String(), test.expOption) {
|
||||
t.Errorf("expected option: '%v', got: '%v'", test.expOption, unkeyedIdx.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
477
idk/api/source.go
Normal file
477
idk/api/source.go
Normal file
|
|
@ -0,0 +1,477 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"strconv"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
pilosaclient "github.com/molecula/featurebase/v3/client"
|
||||
"github.com/molecula/featurebase/v3/pql"
|
||||
)
|
||||
|
||||
func IngestJSON(index *pilosaclient.Index, m idk.Main, in io.Reader) error {
|
||||
codec, err := newJSONCodec(in)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "reading start of JSON data")
|
||||
}
|
||||
|
||||
fields := index.Fields()
|
||||
|
||||
idkSchema := make([]idk.Field, 1, len(index.Fields())+1)
|
||||
fieldMappers := make(map[string]mapper, len(fields))
|
||||
if index.Opts().Keys() {
|
||||
idkSchema[0] = idk.StringField{
|
||||
NameVal: "/",
|
||||
}
|
||||
m.PrimaryKeyFields = []string{"/"}
|
||||
} else {
|
||||
idkSchema[0] = idk.IDField{
|
||||
NameVal: "/",
|
||||
}
|
||||
m.IDField = "/"
|
||||
}
|
||||
|
||||
m.Index = index.Name()
|
||||
|
||||
// TODO: time quantums
|
||||
for name, field := range fields {
|
||||
i := len(idkSchema)
|
||||
fopts := field.Opts()
|
||||
switch typ := fopts.Type(); typ {
|
||||
case pilosaclient.FieldTypeSet:
|
||||
if fopts.Keys() {
|
||||
idkSchema = append(idkSchema, idk.StringArrayField{
|
||||
NameVal: name,
|
||||
})
|
||||
fieldMappers[name] = mapper{
|
||||
idx: i,
|
||||
mapper: func(v interface{}) (interface{}, error) {
|
||||
var set []string
|
||||
switch v := v.(type) {
|
||||
case string:
|
||||
set = []string{v}
|
||||
|
||||
case []interface{}:
|
||||
set = make([]string, len(v))
|
||||
for i, elem := range v {
|
||||
str, ok := elem.(string)
|
||||
if !ok {
|
||||
return nil, errors.Wrapf(TypeError{
|
||||
Expected: typeDescriptionString,
|
||||
Value: elem,
|
||||
}, "parsing element %d of set", i)
|
||||
}
|
||||
|
||||
set[i] = str
|
||||
}
|
||||
|
||||
if len(set) > 1 {
|
||||
dups := make(map[string]struct{}, len(set))
|
||||
for _, elem := range set {
|
||||
if _, dup := dups[elem]; dup {
|
||||
return nil, ErrDuplicateElement{
|
||||
Elem: elem,
|
||||
}
|
||||
}
|
||||
dups[elem] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
return nil, TypeError{
|
||||
Expected: typeDescriptionStringSet,
|
||||
Value: v,
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range set {
|
||||
if v == "" {
|
||||
return nil, errors.New("empty string in set")
|
||||
}
|
||||
}
|
||||
|
||||
return set, nil
|
||||
},
|
||||
}
|
||||
} else {
|
||||
idkSchema = append(idkSchema, idk.IDArrayField{
|
||||
NameVal: name,
|
||||
})
|
||||
fieldMappers[name] = mapper{
|
||||
idx: i,
|
||||
mapper: func(v interface{}) (interface{}, error) {
|
||||
var set []uint64
|
||||
switch v := v.(type) {
|
||||
case json.Number:
|
||||
id, err := strconv.ParseUint(string(v), 10, 64)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "parsing ID")
|
||||
}
|
||||
|
||||
set = []uint64{id}
|
||||
|
||||
case []interface{}:
|
||||
set = make([]uint64, len(v))
|
||||
for i, elem := range v {
|
||||
num, ok := elem.(json.Number)
|
||||
if !ok {
|
||||
return nil, errors.Wrapf(TypeError{
|
||||
Expected: typeDescriptionID,
|
||||
Value: elem,
|
||||
}, "parsing element %d of set", i)
|
||||
}
|
||||
|
||||
id, err := strconv.ParseUint(string(num), 10, 64)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "parsing element %d of set", i)
|
||||
}
|
||||
|
||||
set[i] = id
|
||||
}
|
||||
|
||||
if len(set) > 1 {
|
||||
dups := make(map[uint64]struct{}, len(set))
|
||||
for _, elem := range set {
|
||||
if _, dup := dups[elem]; dup {
|
||||
return nil, ErrDuplicateElement{
|
||||
Elem: elem,
|
||||
}
|
||||
}
|
||||
dups[elem] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
return nil, TypeError{
|
||||
Expected: typeDescriptionIDSet,
|
||||
Value: v,
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range set {
|
||||
if v == ^uint64(0) {
|
||||
// The client package uses this as a nil sentinel.
|
||||
return nil, errors.New("max uint64 is not a valid row ID")
|
||||
}
|
||||
}
|
||||
|
||||
return set, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
case pilosaclient.FieldTypeMutex:
|
||||
if fopts.Keys() {
|
||||
idkSchema = append(idkSchema, idk.StringField{
|
||||
NameVal: name,
|
||||
Mutex: true,
|
||||
})
|
||||
fieldMappers[name] = mapper{
|
||||
idx: i,
|
||||
mapper: func(v interface{}) (interface{}, error) {
|
||||
str, ok := v.(string)
|
||||
if !ok {
|
||||
return nil, TypeError{
|
||||
Expected: typeDescriptionString,
|
||||
Value: v,
|
||||
}
|
||||
}
|
||||
|
||||
return str, nil
|
||||
},
|
||||
}
|
||||
} else {
|
||||
idkSchema = append(idkSchema, idk.IDField{
|
||||
NameVal: name,
|
||||
Mutex: true,
|
||||
})
|
||||
fieldMappers[name] = mapper{
|
||||
idx: i,
|
||||
mapper: func(v interface{}) (interface{}, error) {
|
||||
num, ok := v.(json.Number)
|
||||
if !ok {
|
||||
return nil, TypeError{
|
||||
Expected: typeDescriptionID,
|
||||
Value: v,
|
||||
}
|
||||
}
|
||||
|
||||
id, err := strconv.ParseUint(string(num), 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if id == ^uint64(0) {
|
||||
// The client package uses this as a nil sentinel.
|
||||
return nil, errors.New("max uint64 is not a valid row ID")
|
||||
}
|
||||
|
||||
return id, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
case pilosaclient.FieldTypeBool:
|
||||
// TODO: not supported by IDK :)
|
||||
idkSchema = append(idkSchema, idk.BoolField{
|
||||
NameVal: name,
|
||||
})
|
||||
*/
|
||||
|
||||
case pilosaclient.FieldTypeInt:
|
||||
if fopts.Keys() {
|
||||
idkSchema = append(idkSchema, idk.StringField{
|
||||
NameVal: name,
|
||||
})
|
||||
fieldMappers[name] = mapper{
|
||||
idx: i,
|
||||
mapper: func(v interface{}) (interface{}, error) {
|
||||
str, ok := v.(string)
|
||||
if !ok {
|
||||
return nil, TypeError{
|
||||
Expected: typeDescriptionString,
|
||||
Value: v,
|
||||
}
|
||||
}
|
||||
|
||||
return str, nil
|
||||
},
|
||||
}
|
||||
break
|
||||
}
|
||||
idkSchema = append(idkSchema, idk.IntField{
|
||||
NameVal: name,
|
||||
})
|
||||
fieldMappers[name] = mapper{
|
||||
idx: i,
|
||||
mapper: func(v interface{}) (interface{}, error) {
|
||||
number, ok := v.(json.Number)
|
||||
if !ok {
|
||||
return nil, TypeError{
|
||||
Expected: typeDescriptionInt,
|
||||
Value: v,
|
||||
}
|
||||
}
|
||||
|
||||
return strconv.ParseInt(string(number), 10, 64)
|
||||
},
|
||||
}
|
||||
|
||||
case pilosaclient.FieldTypeDecimal:
|
||||
scale := fopts.Scale()
|
||||
idkSchema = append(idkSchema, idk.DecimalField{
|
||||
NameVal: name,
|
||||
Scale: scale,
|
||||
})
|
||||
fieldMappers[name] = mapper{
|
||||
idx: i,
|
||||
mapper: func(v interface{}) (interface{}, error) {
|
||||
var raw string
|
||||
switch v := v.(type) {
|
||||
case json.Number:
|
||||
raw = string(v)
|
||||
case string:
|
||||
raw = string(v)
|
||||
default:
|
||||
return nil, TypeError{
|
||||
Expected: typeDescriptionDecimal,
|
||||
Value: v,
|
||||
}
|
||||
}
|
||||
|
||||
dec, err := pql.ParseDecimal(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return dec, nil
|
||||
},
|
||||
}
|
||||
|
||||
/*
|
||||
// Pilosa unfortunately does not return the epoch to us.
|
||||
// As a result, this does not currently work.
|
||||
// Also now we need to parse the timestamp here.
|
||||
case pilosaclient.FieldTypeTimestamp:
|
||||
idkSchema = append(idkSchema, idk.TimestampField{
|
||||
NameVal: name,
|
||||
Epoch: fopts.MinTimestamp(),
|
||||
Granularity: fopts.TimeUnit(),
|
||||
Layout: time.RFC3339Nano,
|
||||
})
|
||||
fieldMappers[name] = mapper{
|
||||
idx: i,
|
||||
mapper: func(v interface{}) (interface{}, error) {
|
||||
str, ok := v.(string)
|
||||
if !ok {
|
||||
return nil, TypeError{
|
||||
Expected: "time string",
|
||||
Value: v,
|
||||
}
|
||||
}
|
||||
|
||||
return str, nil
|
||||
},
|
||||
}*/
|
||||
|
||||
default:
|
||||
// Ignore the field for now.
|
||||
}
|
||||
}
|
||||
|
||||
src := source{
|
||||
c: codec,
|
||||
fields: fieldMappers,
|
||||
schema: idkSchema,
|
||||
pkstring: index.Opts().Keys(),
|
||||
}
|
||||
m.NewSource = func() (idk.Source, error) {
|
||||
return &src, nil
|
||||
}
|
||||
|
||||
return m.Run()
|
||||
}
|
||||
|
||||
// TypeError is an error indicating that the type of a value was set incorrectly.
|
||||
type TypeError struct {
|
||||
// Expected is a human-readable name of the expected type.
|
||||
Expected string
|
||||
|
||||
// Value is the incorrectly-typed value.
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
const (
|
||||
typeDescriptionID = "ID"
|
||||
typeDescriptionIDSet = "set of " + typeDescriptionID + "s"
|
||||
typeDescriptionString = "string"
|
||||
typeDescriptionStringSet = "set of " + typeDescriptionString + "s"
|
||||
typeDescriptionInt = "integer"
|
||||
typeDescriptionDecimal = "decimal"
|
||||
)
|
||||
|
||||
func (t TypeError) Error() string {
|
||||
valType := reflect.TypeOf(t.Value)
|
||||
valTypeName := valType.String()
|
||||
if name, ok := friendlyTypeNames[valType]; ok {
|
||||
valTypeName = name
|
||||
}
|
||||
|
||||
return fmt.Sprintf("expected a %s but got %v (a %s)", t.Expected, t.Value, valTypeName)
|
||||
}
|
||||
|
||||
// friendlyTypeNames contains more-easily understood names for some common types.
|
||||
var friendlyTypeNames = map[reflect.Type]string{
|
||||
reflect.TypeOf(json.Number("")): "number",
|
||||
reflect.TypeOf([]interface{}{}): "array",
|
||||
reflect.TypeOf(map[string]interface{}{}): "object",
|
||||
}
|
||||
|
||||
// ErrDuplicateElement is an error indicating that an element was included in a set multiple times.
|
||||
type ErrDuplicateElement struct {
|
||||
// Elem is the duplicated element.
|
||||
Elem interface{}
|
||||
}
|
||||
|
||||
func (err ErrDuplicateElement) Error() string {
|
||||
return fmt.Sprintf("found duplicate element %v in set", err.Elem)
|
||||
}
|
||||
|
||||
type source struct {
|
||||
c *jsonCodec
|
||||
pkstring bool
|
||||
fields map[string]mapper
|
||||
schema []idk.Field
|
||||
}
|
||||
|
||||
type mapper struct {
|
||||
idx int
|
||||
mapper func(interface{}) (interface{}, error)
|
||||
}
|
||||
|
||||
func (s *source) Record() (idk.Record, error) {
|
||||
// Decode the next record.
|
||||
raw, err := s.c.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Parse the record into IDK-format.
|
||||
out := make(idkRec, len(s.schema))
|
||||
switch pk := raw.PrimaryKey.(type) {
|
||||
case string:
|
||||
if !s.pkstring {
|
||||
return nil, errors.Wrap(TypeError{
|
||||
Expected: typeDescriptionID,
|
||||
Value: pk,
|
||||
}, "parsing primary key")
|
||||
}
|
||||
|
||||
out[0] = pk
|
||||
|
||||
case json.Number:
|
||||
if s.pkstring {
|
||||
return nil, errors.Wrap(TypeError{
|
||||
Expected: typeDescriptionString,
|
||||
Value: pk,
|
||||
}, "parsing primary key")
|
||||
}
|
||||
|
||||
id, err := strconv.ParseUint(string(pk), 10, 64)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "parsing primary key")
|
||||
}
|
||||
|
||||
out[0] = id
|
||||
|
||||
default:
|
||||
expect := typeDescriptionID
|
||||
if s.pkstring {
|
||||
expect = typeDescriptionString
|
||||
}
|
||||
return nil, errors.Wrap(TypeError{
|
||||
Expected: expect,
|
||||
Value: pk,
|
||||
}, "parsing primary key")
|
||||
}
|
||||
for field, value := range raw.Values {
|
||||
m, ok := s.fields[field]
|
||||
if !ok {
|
||||
return nil, errors.Errorf("field %q missing from schema", field)
|
||||
}
|
||||
|
||||
v, err := m.mapper(value)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "parsing record field %q", field)
|
||||
}
|
||||
|
||||
out[m.idx] = v
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *source) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *source) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type idkRec []interface{}
|
||||
|
||||
func (r idkRec) Commit(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r idkRec) Data() []interface{} {
|
||||
return r
|
||||
}
|
||||
225
idk/api/source_test.go
Normal file
225
idk/api/source_test.go
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
pilosaclient "github.com/molecula/featurebase/v3/client"
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIngest(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip()
|
||||
}
|
||||
|
||||
client, err := pilosaclient.NewClient("pilosa:10101")
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
err := client.Close()
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
schema, err := client.Schema()
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
if schema.HasIndex("apitestingest_unkeyed") {
|
||||
err := client.DeleteIndexByName("apitestingest_unkeyed")
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
schema, err = client.Schema()
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
|
||||
unkeyedIdx := schema.Index("apitestingest_unkeyed", pilosaclient.OptIndexTrackExistence(true))
|
||||
unkeyedIdx.Field("idset")
|
||||
unkeyedIdx.Field("id", pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0))
|
||||
unkeyedIdx.Field("stringset", pilosaclient.OptFieldKeys(true))
|
||||
unkeyedIdx.Field("string", pilosaclient.OptFieldKeys(true), pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0))
|
||||
//unkeyedIdx.Field("bool", pilosaclient.OptFieldTypeBool())
|
||||
unkeyedIdx.Field("int", pilosaclient.OptFieldTypeInt())
|
||||
unkeyedIdx.Field("decimal", pilosaclient.OptFieldTypeDecimal(2))
|
||||
//unkeyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s"))
|
||||
keyedIdx := schema.Index("apitestingest_keyed", pilosaclient.OptIndexTrackExistence(true), pilosaclient.OptIndexKeys(true))
|
||||
keyedIdx.Field("idset")
|
||||
keyedIdx.Field("id", pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0))
|
||||
keyedIdx.Field("stringset", pilosaclient.OptFieldKeys(true))
|
||||
keyedIdx.Field("string", pilosaclient.OptFieldKeys(true), pilosaclient.OptFieldTypeMutex(pilosaclient.CacheTypeNone, 0))
|
||||
//keyedIdx.Field("bool", pilosaclient.OptFieldTypeBool())
|
||||
keyedIdx.Field("int", pilosaclient.OptFieldTypeInt())
|
||||
keyedIdx.Field("decimal", pilosaclient.OptFieldTypeDecimal(2))
|
||||
//keyedIdx.Field("timestamp", pilosaclient.OptFieldTypeTimestamp(time.Unix(1, 0).UTC(), "s"))
|
||||
|
||||
err = client.SyncSchema(schema)
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
err := client.DeleteIndex(unkeyedIdx)
|
||||
assert.NoError(t, err)
|
||||
err = client.DeleteIndex(keyedIdx)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
m := *idk.NewMain()
|
||||
m.PilosaHosts = []string{"pilosa:10101"}
|
||||
m.PilosaGRPCHosts = []string{"pilosa:20101"}
|
||||
m.Verbose = true
|
||||
m.Pprof = ""
|
||||
m.Stats = ""
|
||||
m.PackBools = ""
|
||||
|
||||
t.Run("Unkeyed", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := IngestJSON(unkeyedIdx, m, strings.NewReader(`[
|
||||
{
|
||||
"primary-key": 0,
|
||||
"values": {
|
||||
"idset": [5, 7],
|
||||
"id": 4,
|
||||
"stringset": ["a"],
|
||||
"string": "h",
|
||||
"int": -13,
|
||||
"decimal": 1.01
|
||||
}
|
||||
},
|
||||
{
|
||||
"primary-key": 1,
|
||||
"values": {
|
||||
"idset": 6,
|
||||
"id": 8,
|
||||
"stringset": "b",
|
||||
"decimal": "1.02"
|
||||
}
|
||||
},
|
||||
{
|
||||
"primary-key": 2,
|
||||
"values": {}
|
||||
}
|
||||
]`))
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
|
||||
// Unfortunately the go-pilosa client is still very behind.
|
||||
resp, err := http.Post("http://pilosa:10101/index/apitestingest_unkeyed/query", "text/plain", strings.NewReader("Extract(All(), Rows(idset), Rows(id), Rows(stringset), Rows(string), Rows(int), Rows(decimal))"))
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck
|
||||
if !assert.Equal(t, 200, resp.StatusCode) {
|
||||
body, _ := ioutil.ReadAll(resp.Body)
|
||||
t.Logf("request error: %s", body)
|
||||
return
|
||||
}
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
|
||||
expect := `{
|
||||
"results": [
|
||||
{
|
||||
"fields": [
|
||||
{"name":"idset","type":"[]uint64"},
|
||||
{"name":"id","type":"uint64"},
|
||||
{"name":"stringset","type":"[]string"},
|
||||
{"name":"string","type":"string"},
|
||||
{"name":"int","type":"int64"},
|
||||
{"name":"decimal","type":"decimal"}
|
||||
],
|
||||
"columns": [
|
||||
{"column":0,"rows":[[5,7],4,["a"],"h",-13,1.01]},
|
||||
{"column":1,"rows":[[6],8,["b"],null,null,1.02]},
|
||||
{"column":2,"rows":[[],null,[],null,null,null]}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`
|
||||
assert.JSONEq(t, expect, string(data))
|
||||
})
|
||||
t.Run("Keyed", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := IngestJSON(keyedIdx, m, strings.NewReader(`[
|
||||
{
|
||||
"primary-key": "a",
|
||||
"values": {
|
||||
"idset": [5, 7],
|
||||
"id": 4,
|
||||
"stringset": ["a"],
|
||||
"string": "h",
|
||||
"int": -13,
|
||||
"decimal": 1.01
|
||||
}
|
||||
},
|
||||
{
|
||||
"primary-key": "b",
|
||||
"values": {
|
||||
"idset": 6,
|
||||
"id": 8,
|
||||
"stringset": "b",
|
||||
"decimal": "1.02"
|
||||
}
|
||||
},
|
||||
{
|
||||
"primary-key": "c",
|
||||
"values": {}
|
||||
}
|
||||
]`))
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
|
||||
// Unfortunately the go-pilosa client is still very behind.
|
||||
resp, err := http.Post("http://pilosa:10101/index/apitestingest_keyed/query", "text/plain", strings.NewReader("Extract(All(), Rows(idset), Rows(id), Rows(stringset), Rows(string), Rows(int), Rows(decimal))"))
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer io.Copy(ioutil.Discard, resp.Body) //nolint: errcheck
|
||||
if !assert.Equal(t, 200, resp.StatusCode) {
|
||||
body, _ := ioutil.ReadAll(resp.Body)
|
||||
t.Logf("request error: %s", body)
|
||||
return
|
||||
}
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
|
||||
expect := `{
|
||||
"results": [
|
||||
{
|
||||
"fields": [
|
||||
{"name":"idset","type":"[]uint64"},
|
||||
{"name":"id","type":"uint64"},
|
||||
{"name":"stringset","type":"[]string"},
|
||||
{"name":"string","type":"string"},
|
||||
{"name":"int","type":"int64"},
|
||||
{"name":"decimal","type":"decimal"}
|
||||
],
|
||||
"columns": [
|
||||
{"column":"a","rows":[[5,7],4,["a"],"h",-13,1.01]},
|
||||
{"column":"b","rows":[[6],8,["b"],null,null,1.02]},
|
||||
{"column":"c","rows":[[],null,[],null,null,null]}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`
|
||||
assert.JSONEq(t, expect, string(data))
|
||||
})
|
||||
}
|
||||
315
idk/bankgen/put.go
Normal file
315
idk/bankgen/put.go
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
package bankgen
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
liavro "github.com/linkedin/goavro/v2"
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/common"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka/csrc"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
confluent "github.com/confluentinc/confluent-kafka-go/kafka"
|
||||
)
|
||||
|
||||
// PutCmd represents a command to put data into Kafka
|
||||
type PutCmd struct {
|
||||
idk.ConfluentCommand `flag:"!embed"`
|
||||
|
||||
Topic string `help:"Kafka topic to post to."`
|
||||
Subject string
|
||||
Seed int64 `help:"Random seed for data generation"`
|
||||
Save string `help:"Filename of record log. '-' for stdout."`
|
||||
BatchSize int `help:"Size of record batches to submit to Kafka."`
|
||||
|
||||
NumRecords int `help:"Number of records to generate"`
|
||||
|
||||
DryRun bool `help:"Dry run - just flag parsing."`
|
||||
|
||||
log logger.Logger
|
||||
rand *rand.Rand
|
||||
configMap *confluent.ConfigMap
|
||||
}
|
||||
|
||||
// NewPutCmd creates a new PutCmd
|
||||
func NewPutCmd() (*PutCmd, error) {
|
||||
var err error
|
||||
p := &PutCmd{}
|
||||
p.ConfluentCommand = idk.ConfluentCommand{
|
||||
KafkaBootstrapServers: []string{"localhost:9092"},
|
||||
SchemaRegistryURL: "localhost:8081",
|
||||
}
|
||||
|
||||
p.Topic = "banktest"
|
||||
p.Subject = "test"
|
||||
p.Save = ""
|
||||
p.BatchSize = 1000
|
||||
p.NumRecords = 10
|
||||
p.log = logger.NewVerboseLogger(os.Stderr)
|
||||
return p, err
|
||||
}
|
||||
|
||||
func (p *PutCmd) Log() logger.Logger { return p.log }
|
||||
|
||||
// Run is the entry point for PutCmd
|
||||
func (p *PutCmd) Run() (err error) {
|
||||
p.configMap, err = common.SetupConfluent(&p.ConfluentCommand)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var auth *csrc.BasicAuth
|
||||
if len(p.SchemaRegistryUsername) > 0 {
|
||||
auth = &csrc.BasicAuth{
|
||||
KafkaSchemaApiKey: p.SchemaRegistryUsername,
|
||||
KafkaSchemaApiSecret: p.SchemaRegistryPassword,
|
||||
}
|
||||
}
|
||||
client := csrc.NewClient(p.SchemaRegistryURL, nil, auth)
|
||||
|
||||
schemaStr := getBankSchema()
|
||||
|
||||
resp, err := client.PostSubjects(p.Subject, schemaStr)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "posting schema")
|
||||
}
|
||||
p.log.Printf("Posted schema ID: %d\n", resp.ID)
|
||||
|
||||
licodec, err := liavro.NewCodec(schemaStr)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "li decoding schema")
|
||||
}
|
||||
|
||||
p.rand = rand.New(rand.NewSource(p.Seed))
|
||||
|
||||
var w io.Writer = os.Stdout
|
||||
if p.Save != "" && p.Save != "-" {
|
||||
f, err := os.Create(p.Save)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "creating record log file")
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
w = f
|
||||
}
|
||||
|
||||
// librdkafka has a large (1000000) default buffer.
|
||||
// limit the go buffer size to reduce memory consumption
|
||||
// but allows for the intended buffered channel behavior
|
||||
err = p.configMap.SetKey("go.produce.channel.size", 1)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "setting go.produce.channel.size")
|
||||
}
|
||||
producer, err := confluent.NewProducer(p.configMap)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to create producer: %s\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer producer.Close()
|
||||
|
||||
finished := int32(0)
|
||||
iter := int64(0)
|
||||
doneChan := common.LaunchKafkaEventConfirmer(producer, &finished, &iter)
|
||||
for n := 0; n < p.NumRecords; n++ {
|
||||
data := p.generateRecord()
|
||||
rec, _, err := licodec.NativeFromTextual([]byte(data))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "decoding generated data")
|
||||
}
|
||||
|
||||
buf := make([]byte, 5, 1000)
|
||||
buf[0] = 0
|
||||
binary.BigEndian.PutUint32(buf[1:], uint32(resp.ID))
|
||||
buf, err = licodec.BinaryFromNative(buf, rec.(map[string]interface{}))
|
||||
if err != nil {
|
||||
return errors.Errorf("encoding:\n%+v\nerr: %v", rec, err)
|
||||
}
|
||||
|
||||
var uid int
|
||||
switch v := rec.(map[string]interface{})["user_id"].(type) {
|
||||
case int:
|
||||
uid = v
|
||||
case int32:
|
||||
uid = int(v)
|
||||
}
|
||||
producer.ProduceChannel() <- &confluent.Message{
|
||||
TopicPartition: confluent.TopicPartition{Topic: &p.Topic, Partition: confluent.PartitionAny},
|
||||
Key: []byte(strconv.Itoa(uid)),
|
||||
Value: buf,
|
||||
}
|
||||
atomic.AddInt64(&iter, 1)
|
||||
|
||||
if p.Save != "" {
|
||||
fmt.Fprintf(w, "%v\n", rec.(map[string]interface{}))
|
||||
}
|
||||
|
||||
}
|
||||
atomic.AddInt32(&finished, 1)
|
||||
<-doneChan //wait till all messages are acked
|
||||
producer.Flush(10 * 1000)
|
||||
p.log.Printf("Put %d generated records\n", iter)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
floatMeta = `"type": ["null", {"type": "float", "scale": 3}], "default": null`
|
||||
boolMeta = `"type": ["null", "boolean"], "default": null`
|
||||
stringMeta = `"type": ["null", "string"], "default": null, "mutex": true`
|
||||
intMeta = `"type": ["null", "int"], "default": null`
|
||||
|
||||
floatNames = []string{
|
||||
"pfm_category_total_current_balance__personal_loan",
|
||||
}
|
||||
|
||||
boolNames = []string{
|
||||
"pfm_boolean__personal_loan",
|
||||
"pfm_boolean__mortgage",
|
||||
}
|
||||
|
||||
stringNames = []string{
|
||||
"survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab",
|
||||
}
|
||||
|
||||
intNames = []string{
|
||||
"product_recency__consumer_loan",
|
||||
}
|
||||
|
||||
floatCount = 1
|
||||
boolCount = 2
|
||||
stringCount = 1
|
||||
intCount = 1
|
||||
|
||||
floatNullChance = 0.1
|
||||
boolNullChance = 0.1
|
||||
stringNullChance = 0.1
|
||||
intNullChance = 0.1
|
||||
)
|
||||
|
||||
func getBankSchema() string {
|
||||
meta := `"type": "record",
|
||||
"name": "bank_user_data",
|
||||
"namespace": "bank.user.data",
|
||||
"doc": "Per-user bank data"`
|
||||
|
||||
fields := []string{
|
||||
`{"name": "aba", "type": "string"}`,
|
||||
`{"name": "db", "type": "string"}`,
|
||||
`{"name": "user_id", "type": "int"}`,
|
||||
`{"name": "custom_audiences", "type": ["null", "string"], "default": null}`,
|
||||
}
|
||||
|
||||
for n := 0; n < floatCount; n++ {
|
||||
fields = append(fields, `{"name": "float_`+floatNames[n]+`", `+floatMeta+`}`)
|
||||
}
|
||||
for n := 0; n < boolCount; n++ {
|
||||
fields = append(fields, `{"name": "bool_`+boolNames[n]+`", `+boolMeta+`}`)
|
||||
}
|
||||
for n := 0; n < stringCount; n++ {
|
||||
fields = append(fields, `{"name": "string_`+stringNames[n]+`", `+stringMeta+`}`)
|
||||
}
|
||||
for n := 0; n < intCount; n++ {
|
||||
fields = append(fields, `{"name": "int_`+intNames[n]+`", `+intMeta+`}`)
|
||||
}
|
||||
schema := fmt.Sprintf(`{%s, "fields": [%s]}`, meta, strings.Join(fields, ", "))
|
||||
|
||||
return schema
|
||||
}
|
||||
|
||||
func (p *PutCmd) generateRecord() string {
|
||||
fields := []string{
|
||||
`"aba": "` + p.ABA() + `"`,
|
||||
`"db": "` + p.Db() + `"`,
|
||||
`"user_id": ` + strconv.FormatInt(int64(p.UserID()), 10),
|
||||
`"custom_audiences": {"string": "` + p.CustomAudiences() + `"}`,
|
||||
}
|
||||
|
||||
for n := 0; n < floatCount; n++ {
|
||||
if floatNullChance < p.rand.Float64() {
|
||||
floatStr := strconv.FormatFloat(p.rand.Float64(), 'f', 3, 64)
|
||||
fields = append(fields, `"float_`+floatNames[n]+`": {"float": `+floatStr+`}`)
|
||||
}
|
||||
}
|
||||
for n := 0; n < boolCount; n++ {
|
||||
if boolNullChance < p.rand.Float64() {
|
||||
boolStr := strconv.FormatBool(p.rand.Intn(2) == 1)
|
||||
fields = append(fields, `"bool_`+boolNames[n]+`": {"boolean": `+boolStr+`}`)
|
||||
}
|
||||
}
|
||||
|
||||
for n := 0; n < stringCount; n++ {
|
||||
if stringNullChance < p.rand.Float64() {
|
||||
stringStr := text(p.rand, 1, 6, true, true, true, false)
|
||||
fields = append(fields, `"string_`+stringNames[n]+`": {"string": "`+stringStr+`"}`)
|
||||
}
|
||||
}
|
||||
|
||||
for n := 0; n < intCount; n++ {
|
||||
if intNullChance < p.rand.Float64() {
|
||||
intStr := strconv.FormatInt(int64(p.rand.Intn(1000000)), 10)
|
||||
fields = append(fields, `"int_`+intNames[n]+`": {"int": `+intStr+`}`)
|
||||
}
|
||||
}
|
||||
|
||||
return `{` + strings.Join(fields, ", ") + `}`
|
||||
}
|
||||
|
||||
// ABA returns a random 9 numeric digit string with about 27000 possible values.
|
||||
func (p *PutCmd) ABA() string {
|
||||
num := p.rand.Intn(27000) + 22213
|
||||
num2 := num/10 - 1213
|
||||
numstr := strconv.Itoa(num)
|
||||
num2str := strconv.Itoa(num2)
|
||||
numstrbytes := append([]byte(numstr), num2str[3], numstr[0], numstr[1], numstr[2])
|
||||
return string(numstrbytes)
|
||||
}
|
||||
|
||||
// Db returns a db
|
||||
func (p *PutCmd) Db() string {
|
||||
return text(p.rand, 1, 6, true, true, true, false)
|
||||
}
|
||||
|
||||
// UserID returns a user ID
|
||||
func (p *PutCmd) UserID() int {
|
||||
return p.rand.Intn(10000000) // 10 mil
|
||||
}
|
||||
|
||||
// CustomAudiences returns a fake Custom Audience string
|
||||
func (p *PutCmd) CustomAudiences() string {
|
||||
return text(p.rand, 1, 6, true, true, true, false)
|
||||
}
|
||||
|
||||
var lowerLetters = []rune("abcdefghijklmnopqrstuvwxyz")
|
||||
var upperLetters = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
||||
var numeric = []rune("0123456789")
|
||||
var specialChars = []rune(`!'@#$%^&*()_+-=[]{};:",./?`)
|
||||
|
||||
func text(rand *rand.Rand, atLeast, atMost int, allowLower, allowUpper, allowNumeric, allowSpecial bool) string {
|
||||
allowedChars := []rune{}
|
||||
if allowLower {
|
||||
allowedChars = append(allowedChars, lowerLetters...)
|
||||
}
|
||||
if allowUpper {
|
||||
allowedChars = append(allowedChars, upperLetters...)
|
||||
}
|
||||
if allowNumeric {
|
||||
allowedChars = append(allowedChars, numeric...)
|
||||
}
|
||||
if allowSpecial {
|
||||
allowedChars = append(allowedChars, specialChars...)
|
||||
}
|
||||
|
||||
result := []rune{}
|
||||
nTimes := rand.Intn(atMost-atLeast+1) + atLeast
|
||||
for i := 0; i < nTimes; i++ {
|
||||
result = append(result, allowedChars[rand.Intn(len(allowedChars))])
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
26
idk/bankgen/put_test.go
Normal file
26
idk/bankgen/put_test.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
package bankgen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBankgen(t *testing.T) {
|
||||
p, err := NewPutCmd()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get NewPutCmd: %v", err)
|
||||
}
|
||||
|
||||
p.KafkaBootstrapServers = []string{"kafka:9092"}
|
||||
p.SchemaRegistryURL = "http://schema-registry:8081"
|
||||
p.Topic = fmt.Sprintf("topic-%d", rand.New(rand.NewSource(time.Now().Unix())).Uint64())
|
||||
p.NumRecords = 100
|
||||
|
||||
err = p.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to run bankgen: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
35
idk/cmd/bankgen/main.go
Normal file
35
idk/cmd/bankgen/main.go
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/bankgen"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
m, err := bankgen.NewPutCmd()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := pflag.LoadEnv(m, "BANKGEN_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
|
||||
if err := m.Run(); err != nil {
|
||||
log := m.Log()
|
||||
if log == nil {
|
||||
// if we fail before a logger was instantiated
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
129
idk/cmd/bankgen/main_test.go
Normal file
129
idk/cmd/bankgen/main_test.go
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/jaffee/commandeer"
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/bankgen"
|
||||
pflag13 "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func TestBankgenArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
|
||||
KafkaHosts []string
|
||||
RegistryURL string
|
||||
Topic string
|
||||
Subject string
|
||||
Save string
|
||||
NumRecords int
|
||||
Seed int64
|
||||
BatchSize int
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{
|
||||
"", // os.Args[0] can be ignored
|
||||
},
|
||||
KafkaHosts: []string{"localhost:9092"},
|
||||
RegistryURL: "localhost:8081",
|
||||
Topic: "banktest",
|
||||
Subject: "test",
|
||||
Save: "",
|
||||
NumRecords: 10,
|
||||
},
|
||||
{
|
||||
name: "many-kafka-hosts",
|
||||
args: []string{
|
||||
"bankgen", // os.Args[0] can be ignored
|
||||
"--kafka-bootstrap-servers", "localhost:0,localhost:80,localhost:9092",
|
||||
"--num-records", "123",
|
||||
"--schema-registry-url", "https://localhost:8081",
|
||||
"--save", "-",
|
||||
"--subject", "test",
|
||||
"--topic", "test",
|
||||
"--batch-size", "30",
|
||||
},
|
||||
KafkaHosts: []string{"localhost:0", "localhost:80", "localhost:9092"},
|
||||
RegistryURL: "https://localhost:8081",
|
||||
Topic: "test",
|
||||
Subject: "test",
|
||||
Save: "-",
|
||||
NumRecords: 123,
|
||||
BatchSize: 30,
|
||||
},
|
||||
{
|
||||
name: "with-seed",
|
||||
args: []string{
|
||||
"bankgen", // os.Args[0] can be ignored
|
||||
"--kafka-bootstrap-servers", "kafka:9092",
|
||||
"--num-records", "10",
|
||||
"--schema-registry-url", "localhost:8081",
|
||||
"--save", "/tmp/file.log",
|
||||
"--seed", "1234567890",
|
||||
"--subject", "subject",
|
||||
"--topic", "banktest",
|
||||
},
|
||||
KafkaHosts: []string{"kafka:9092"},
|
||||
RegistryURL: "localhost:8081",
|
||||
Topic: "banktest",
|
||||
Subject: "subject",
|
||||
Save: "/tmp/file.log",
|
||||
NumRecords: 10,
|
||||
Seed: int64(1234567890),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs := &pflag.FlagSet{FlagSet: pflag13.NewFlagSet(tc.args[0], pflag13.ExitOnError)}
|
||||
m, err := bankgen.NewPutCmd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := commandeer.LoadArgsEnv(fs, m, tc.args[1:], "BANKGEN_", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(tc.KafkaHosts, m.KafkaBootstrapServers) {
|
||||
t.Fatalf("--kafka-bootstrap-servers expected: %v got: %v", tc.KafkaHosts, m.KafkaBootstrapServers)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.RegistryURL, m.SchemaRegistryURL) {
|
||||
t.Fatalf("--schema-registry-url expected: %v got: %v", tc.RegistryURL, m.SchemaRegistryURL)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(tc.Topic, m.Topic) {
|
||||
t.Fatalf("--topic expected: %v got: %v", tc.Topic, m.Topic)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(tc.Subject, m.Subject) {
|
||||
t.Fatalf("--subject expected: %v got: %v", tc.Subject, m.Subject)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(tc.Save, m.Save) {
|
||||
t.Fatalf("--save expected: %v got: %v", tc.Save, m.Save)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(tc.NumRecords, m.NumRecords) {
|
||||
t.Fatalf("--num-records expected: %v got: %v", tc.NumRecords, m.NumRecords)
|
||||
}
|
||||
|
||||
if tc.BatchSize != 0 {
|
||||
if !reflect.DeepEqual(tc.BatchSize, m.BatchSize) {
|
||||
t.Fatalf("--batch-size expected: %v got: %v", tc.BatchSize, m.BatchSize)
|
||||
}
|
||||
}
|
||||
|
||||
if tc.Seed != 0 {
|
||||
if !reflect.DeepEqual(tc.Seed, m.Seed) {
|
||||
t.Fatalf("--seed expected: %v got: %v", tc.Seed, m.Seed)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
52
idk/cmd/datagen/main.go
Normal file
52
idk/cmd/datagen/main.go
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
"gopkg.in/DataDog/dd-trace-go.v1/profiler"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
m := datagen.NewMain()
|
||||
|
||||
if err := pflag.LoadEnv(m, "GEN_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if m.Datadog {
|
||||
err := profiler.Start(
|
||||
profiler.WithService("datagen"),
|
||||
profiler.WithEnv("fb-1253"),
|
||||
profiler.WithVersion("v1"),
|
||||
profiler.WithProfileTypes(
|
||||
profiler.CPUProfile,
|
||||
profiler.HeapProfile,
|
||||
profiler.BlockProfile,
|
||||
profiler.MutexProfile,
|
||||
profiler.GoroutineProfile,
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer profiler.Stop()
|
||||
}
|
||||
|
||||
if err := m.Preload(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
m.PrintPlan()
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
if err := m.Run(); err != nil {
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
174
idk/cmd/datagen/main_test.go
Normal file
174
idk/cmd/datagen/main_test.go
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jaffee/commandeer"
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen"
|
||||
pflag13 "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func TestDatagenArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
|
||||
Source string
|
||||
Target string
|
||||
Concurrency int
|
||||
StartFrom uint64
|
||||
EndAt uint64
|
||||
|
||||
PilosaHosts []string
|
||||
PilosaIndex string
|
||||
PilosaBatchSize int
|
||||
|
||||
KafkaHosts []string
|
||||
KafkaRegistryURL string
|
||||
KafkaTopic string
|
||||
KafkaSubject string
|
||||
KafkaBatchSize int
|
||||
KafkaReplicationFactor int
|
||||
KafkaNumPartitions int
|
||||
Datadog bool
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{
|
||||
"", // os.Args[0] can be ignored
|
||||
"--source", "example",
|
||||
},
|
||||
Source: "example",
|
||||
Target: "featurebase",
|
||||
Concurrency: 1,
|
||||
StartFrom: uint64(0),
|
||||
EndAt: uint64(0),
|
||||
|
||||
PilosaHosts: nil,
|
||||
PilosaIndex: "",
|
||||
PilosaBatchSize: 0,
|
||||
|
||||
KafkaHosts: nil,
|
||||
KafkaRegistryURL: "",
|
||||
KafkaTopic: "",
|
||||
KafkaSubject: "",
|
||||
KafkaReplicationFactor: 0,
|
||||
KafkaNumPartitions: 0,
|
||||
Datadog: false,
|
||||
},
|
||||
{
|
||||
name: "long",
|
||||
args: []string{
|
||||
"datagen",
|
||||
"--source", "customer",
|
||||
"--target", "kafka",
|
||||
"--concurrency", "10",
|
||||
"--start-from", "1",
|
||||
"--end-at", "11",
|
||||
"--pilosa.hosts", "pilosa:1,pilosa:2,pilosa:3",
|
||||
"--pilosa.index", "idx",
|
||||
"--pilosa.batch-size", "12345",
|
||||
"--kafka.confluent-command.kafka-bootstrap-servers", "kafka:1,kafka:2,kafka:3",
|
||||
"--kafka.confluent-command.schema-registry-url", "registry:1",
|
||||
"--kafka.subject", "subj",
|
||||
"--kafka.topic", "top",
|
||||
"--kafka.batch-size", "2",
|
||||
"--kafka.replication-factor", "3",
|
||||
"--kafka.num-partitions", "3",
|
||||
"--datadog",
|
||||
},
|
||||
Source: "customer",
|
||||
Target: "kafka",
|
||||
Concurrency: 10,
|
||||
StartFrom: uint64(1),
|
||||
EndAt: uint64(11),
|
||||
|
||||
PilosaHosts: []string{"pilosa:1", "pilosa:2", "pilosa:3"},
|
||||
PilosaIndex: "idx",
|
||||
PilosaBatchSize: 12345,
|
||||
|
||||
KafkaHosts: []string{"kafka:1", "kafka:2", "kafka:3"},
|
||||
KafkaRegistryURL: "registry:1",
|
||||
KafkaSubject: "subj",
|
||||
KafkaTopic: "top",
|
||||
KafkaBatchSize: 2,
|
||||
KafkaReplicationFactor: 3,
|
||||
KafkaNumPartitions: 3,
|
||||
Datadog: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs := &pflag.FlagSet{FlagSet: pflag13.NewFlagSet(tc.args[0], pflag13.ExitOnError)}
|
||||
m := datagen.NewMain()
|
||||
|
||||
if err := commandeer.LoadArgsEnv(fs, m, tc.args[1:], "GEN_", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := m.Preload(); err != nil {
|
||||
t.Fatalf("failed to load: %v", err)
|
||||
}
|
||||
|
||||
if tc.Source != m.Source {
|
||||
t.Fatalf("--source expected: %v got: %v", tc.Source, m.Source)
|
||||
}
|
||||
if tc.Target != m.Target {
|
||||
t.Fatalf("--target expected: %v got: %v", tc.Target, m.Target)
|
||||
}
|
||||
if tc.Concurrency != m.Concurrency {
|
||||
t.Fatalf("--concurrency expected: %v got: %v", tc.Concurrency, m.Concurrency)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(tc.PilosaHosts, m.Pilosa.Hosts) {
|
||||
t.Fatalf("--pilosa.hosts expected: %v got: %v", tc.PilosaHosts, m.Pilosa.Hosts)
|
||||
}
|
||||
if tc.PilosaIndex != m.Pilosa.Index {
|
||||
t.Fatalf("--pilosa.index expected: %v got: %v", tc.PilosaIndex, m.Pilosa.Index)
|
||||
}
|
||||
if tc.PilosaBatchSize != m.Pilosa.BatchSize {
|
||||
t.Fatalf("--pilosa.batch-size expected: %v got: %v", tc.PilosaBatchSize, m.Pilosa.BatchSize)
|
||||
}
|
||||
|
||||
if tc.KafkaHosts != nil {
|
||||
bs, err := m.KafkaPut.ConfigMap.Get("bootstrap.servers", nil)
|
||||
if err != nil || bs == nil {
|
||||
t.Fatalf("kafka servers not set properly: %v", err)
|
||||
}
|
||||
bootstrapServers := bs.(string)
|
||||
|
||||
if !reflect.DeepEqual(strings.Join(tc.KafkaHosts, ","), bootstrapServers) {
|
||||
t.Fatalf("--kafka.hosts expected: %v got: %v", strings.Join(tc.KafkaHosts, ","), bootstrapServers)
|
||||
}
|
||||
}
|
||||
if tc.KafkaRegistryURL != m.Kafka.SchemaRegistryURL {
|
||||
t.Fatalf("--kafka.registry-url expected: %v got: %v", tc.KafkaRegistryURL, m.Kafka.SchemaRegistryURL)
|
||||
}
|
||||
if tc.KafkaSubject != m.Kafka.Subject {
|
||||
t.Fatalf("--kafka.subject expected: %v got: %v", tc.KafkaSubject, m.Kafka.Subject)
|
||||
}
|
||||
if tc.KafkaTopic != m.Kafka.Topic {
|
||||
t.Fatalf("--kafka.topic expected: %v got: %v", tc.KafkaTopic, m.Kafka.Topic)
|
||||
}
|
||||
if tc.KafkaBatchSize != 0 {
|
||||
if tc.KafkaBatchSize != m.Kafka.BatchSize {
|
||||
t.Fatalf("--kafka.batch-size expected: %v got: %v", tc.KafkaBatchSize, m.Kafka.BatchSize)
|
||||
}
|
||||
}
|
||||
if tc.KafkaReplicationFactor != 0 {
|
||||
if tc.KafkaReplicationFactor != m.Kafka.ReplicationFactor {
|
||||
t.Fatalf("--kafka.replication-factor expected: %v got: %v", tc.KafkaReplicationFactor, m.Kafka.ReplicationFactor)
|
||||
}
|
||||
}
|
||||
if tc.KafkaNumPartitions != 0 {
|
||||
if tc.KafkaNumPartitions != m.Kafka.NumPartitions {
|
||||
t.Fatalf("--kafka.num-partitions expected: %v got: %v", tc.KafkaNumPartitions, m.Kafka.NumPartitions)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
12
idk/cmd/ingester/README.md
Normal file
12
idk/cmd/ingester/README.md
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
## Ingester
|
||||
|
||||
_Ingester service_ consumes [Programmatic Ingest API](https://github.com/molecula/docs/blob/master/docs/internal/proposals/programmatic-ingest-api.md)
|
||||
and sends data to the _Pilosa service_.
|
||||
|
||||
```
|
||||
Usage of ./ingester:
|
||||
--dry-run Dry run - just flag parsing.
|
||||
--http-addr string HTTP address for Ingester. (default "localhost:8080")
|
||||
--pilosa-hosts strings Comma separated list of host:port pairs for Pilosa. (default [localhost:10101])
|
||||
|
||||
```
|
||||
187
idk/cmd/ingester/main.go
Normal file
187
idk/cmd/ingester/main.go
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"runtime/debug"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
pilosaclient "github.com/molecula/featurebase/v3/client"
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/api"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func main() {
|
||||
i := &ingester{
|
||||
PilosaHosts: []string{"localhost:10101"},
|
||||
HttpAddr: "localhost:8080",
|
||||
}
|
||||
|
||||
if err := pflag.LoadEnv(i, "INGESTER_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if i.DryRun {
|
||||
log.Printf("%+v\n", i)
|
||||
return
|
||||
}
|
||||
|
||||
if err := i.Run(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ingester is a HTTP handler which lets ingest data to pilosa over "programmatic ingest API".
|
||||
// https://github.com/molecula/docs/blob/master/docs/internal/proposals/programmatic-ingest-api.md
|
||||
type ingester struct {
|
||||
DryRun bool `help:"Dry run - just flag parsing."`
|
||||
PilosaHosts []string `short:"" help:"Comma separated list of host:port pairs for FeatureBase."`
|
||||
PilosaGrpcHosts []string `short:"" help:"Comma separated list of host:port pairs for FeatureBase's gRPC endpoint."`
|
||||
BatchSize uint `short:"" help:"Batch size to use for ingest operations."`
|
||||
HttpAddr string `short:"" help:"HTTP address for Ingester."`
|
||||
|
||||
client *pilosaclient.Client
|
||||
router *mux.Router
|
||||
tmplMain idk.Main
|
||||
|
||||
AuthToken string `flag:"auth-token" help:"Authentication Token for FeatureBase"`
|
||||
}
|
||||
|
||||
func (i *ingester) Run() error {
|
||||
client, err := pilosaclient.NewClient(i.PilosaHosts,
|
||||
pilosaclient.OptClientRetries(2),
|
||||
pilosaclient.OptClientTotalPoolSize(1000),
|
||||
pilosaclient.OptClientPoolSizePerRoute(400),
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "getting featurebase client")
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
i.client = client
|
||||
|
||||
i.router = mux.NewRouter()
|
||||
i.router.HandleFunc("/schema", i.postSchema).Methods("POST").Name("PostSchema")
|
||||
i.router.HandleFunc("/index/{index}", i.postIndex).Methods("POST").Name("PostIndex")
|
||||
|
||||
m := *idk.NewMain()
|
||||
m.PilosaHosts = i.PilosaHosts
|
||||
m.PilosaGRPCHosts = i.PilosaGrpcHosts
|
||||
m.Pprof = ""
|
||||
m.Stats = ""
|
||||
m.PackBools = ""
|
||||
m.BatchSize = int(i.BatchSize)
|
||||
m.AuthToken = i.AuthToken
|
||||
i.tmplMain = m
|
||||
|
||||
s := &http.Server{Addr: i.HttpAddr, Handler: i}
|
||||
return s.ListenAndServe()
|
||||
}
|
||||
|
||||
// ServeHTTP handles an HTTP request.
|
||||
func (i *ingester) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
stack := debug.Stack()
|
||||
log.Printf("PANIC: %s\n%s", err, stack)
|
||||
}
|
||||
}()
|
||||
|
||||
i.router.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// POST /schema
|
||||
func (i *ingester) postSchema(w http.ResponseWriter, r *http.Request) {
|
||||
body := r.Body
|
||||
defer body.Close()
|
||||
|
||||
schema, ifNotExists, err := api.DecodeSchema(body)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
for idxName, idx := range schema.Indexes() {
|
||||
if err := i.client.CreateIndex(idx); err != nil {
|
||||
if errors.Is(err, pilosaclient.ErrIndexExists) {
|
||||
if ifNotExists[idxName] {
|
||||
continue
|
||||
}
|
||||
|
||||
http.Error(w, err.Error(), http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
for fldName, fld := range idx.Fields() {
|
||||
if fldName == "_exists" {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := i.client.EnsureField(fld); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Following code is not really needed,
|
||||
// but it's kind of informative if we want to check what's the current pilosa schema.
|
||||
// If we decide to keep it, we'll need to improve "pilosa/client",
|
||||
// because with current implementation we can serialize just map of indices.
|
||||
schema, err = i.client.Schema()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte(schema.String())) //nolint: errcheck
|
||||
}
|
||||
|
||||
// POST /index/{index}
|
||||
func (i *ingester) postIndex(w http.ResponseWriter, r *http.Request) {
|
||||
index := mux.Vars(r)["index"]
|
||||
|
||||
schema, err := i.client.Schema()
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("fetching schema: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
idx, ok := schema.Indexes()[index]
|
||||
if !ok {
|
||||
http.Error(w, fmt.Sprintf("index %q not found", index), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
switch ctype := r.Header.Get("Content-Type"); ctype {
|
||||
case "":
|
||||
http.Error(w, "missing Content-Type header", http.StatusUnsupportedMediaType)
|
||||
return
|
||||
|
||||
case "application/json":
|
||||
err = api.IngestJSON(idx, i.tmplMain, r.Body)
|
||||
if err != nil {
|
||||
errCode := http.StatusInternalServerError
|
||||
if errors.As(err, &api.TypeError{}) || errors.As(err, &api.ErrDuplicateElement{}) {
|
||||
errCode = http.StatusBadRequest
|
||||
}
|
||||
http.Error(w, err.Error(), errCode)
|
||||
return
|
||||
}
|
||||
|
||||
default:
|
||||
http.Error(w, fmt.Sprintf("unsupported Content-Type %q", ctype), http.StatusUnsupportedMediaType)
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("OK")) //nolint: errcheck
|
||||
}
|
||||
30
idk/cmd/kafkagen/main.go
Normal file
30
idk/cmd/kafkagen/main.go
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kafkagen"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
m, err := kafkagen.NewMain()
|
||||
if err != nil {
|
||||
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := pflag.LoadEnv(m, "KGEN_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
|
||||
if err := m.Run(); err != nil {
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
90
idk/cmd/kafkagen/main_test.go
Normal file
90
idk/cmd/kafkagen/main_test.go
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/jaffee/commandeer"
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kafkagen"
|
||||
pflag13 "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func TestKafkagenArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
|
||||
KafkaHosts []string
|
||||
RegistryURL string
|
||||
Topic string
|
||||
Subject string
|
||||
SchemaFile string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{
|
||||
"",
|
||||
},
|
||||
|
||||
KafkaHosts: []string{"localhost:9092"},
|
||||
RegistryURL: "localhost:8081",
|
||||
Topic: "defaulttopic",
|
||||
Subject: "bigschema",
|
||||
SchemaFile: "bigschema.json",
|
||||
},
|
||||
{
|
||||
name: "all-set",
|
||||
args: []string{
|
||||
"kafkagen",
|
||||
"--kafka-bootstrap-servers", "kafka:1,kafka:2,kafka:3,kafka4",
|
||||
"--schema-registry-url", "idk.registry.com",
|
||||
"--schema-file", "/tmp/schema.json",
|
||||
"--subject", "whatever",
|
||||
"--topic", "whatever",
|
||||
},
|
||||
KafkaHosts: []string{"kafka:1", "kafka:2", "kafka:3", "kafka4"},
|
||||
RegistryURL: "idk.registry.com",
|
||||
Topic: "whatever",
|
||||
Subject: "whatever",
|
||||
SchemaFile: "/tmp/schema.json",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs := &pflag.FlagSet{FlagSet: pflag13.NewFlagSet(tc.args[0], pflag13.ExitOnError)}
|
||||
m, err := kafkagen.NewMain()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := commandeer.LoadArgsEnv(fs, m, tc.args[1:], "KGEN_", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(tc.KafkaHosts) > 0 {
|
||||
if !reflect.DeepEqual(tc.KafkaHosts, m.KafkaBootstrapServers) {
|
||||
t.Fatalf("--kafka-bootstrap-servers expected: %v got: %v", tc.KafkaHosts, m.KafkaBootstrapServers)
|
||||
}
|
||||
}
|
||||
|
||||
if tc.RegistryURL != "" {
|
||||
if !reflect.DeepEqual(tc.RegistryURL, m.SchemaRegistryURL) {
|
||||
t.Fatalf("--registry-url expected: %v got: %v", tc.RegistryURL, m.SchemaRegistryURL)
|
||||
}
|
||||
}
|
||||
|
||||
if tc.Topic != "" {
|
||||
if !reflect.DeepEqual(tc.Topic, m.Topic) {
|
||||
t.Fatalf("--topic expected: %v got: %v", tc.Topic, m.Topic)
|
||||
}
|
||||
}
|
||||
|
||||
if tc.Subject != "" {
|
||||
if !reflect.DeepEqual(tc.Subject, m.Subject) {
|
||||
t.Fatalf("--subject expected: %v got: %v", tc.Subject, m.Subject)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
34
idk/cmd/kafkaput/main.go
Normal file
34
idk/cmd/kafkaput/main.go
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
m, err := kafka.NewPutCmd()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := pflag.LoadEnv(m, "KPUT_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
if err := m.Run(); err != nil {
|
||||
log := m.Log()
|
||||
if log == nil {
|
||||
// if we fail before a logger was instantiated
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
103
idk/cmd/kafkaput/main_test.go
Normal file
103
idk/cmd/kafkaput/main_test.go
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/jaffee/commandeer"
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka"
|
||||
pflag13 "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func TestKafkaputArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
|
||||
KafkaHosts []string
|
||||
RegistryURL string
|
||||
Topic string
|
||||
Schema string
|
||||
SchemaFile string
|
||||
Subject string
|
||||
Data string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{
|
||||
"",
|
||||
},
|
||||
KafkaHosts: []string{"localhost:9092"},
|
||||
RegistryURL: "localhost:8081",
|
||||
Topic: "defaulttopic",
|
||||
Subject: "test",
|
||||
SchemaFile: "",
|
||||
Schema: "{\"type\": \"record\",\"namespace\": \"c.e\",\"name\": \"F\",\"fields\": [{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"a\",\"type\": \"boolean\"},{\"name\": \"b\", \"type\": \"float\", \"scale\": 2}]}",
|
||||
Data: "{\"id\": 1, \"a\": true, \"b\": 1.43}",
|
||||
},
|
||||
{
|
||||
name: "all-set",
|
||||
args: []string{
|
||||
"kafkaput",
|
||||
"--kafka-bootstrap-servers", "localhost:0,localhost:80,localhost:9092",
|
||||
"--schema-registry-url", "https://localhost:8081",
|
||||
"--topic", "my-topic",
|
||||
"--schema", "{}",
|
||||
"--schema-file", "/tmp/schema.json",
|
||||
"--data", "{}",
|
||||
"--subject", "my-subject",
|
||||
},
|
||||
KafkaHosts: []string{"localhost:0", "localhost:80", "localhost:9092"},
|
||||
RegistryURL: "https://localhost:8081",
|
||||
Topic: "my-topic",
|
||||
Subject: "my-subject",
|
||||
SchemaFile: "/tmp/schema.json",
|
||||
Schema: "{}",
|
||||
Data: "{}",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs := &pflag.FlagSet{FlagSet: pflag13.NewFlagSet(tc.args[0], pflag13.ExitOnError)}
|
||||
m, err := kafka.NewPutCmd()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := commandeer.LoadArgsEnv(fs, m, tc.args[1:], "KPUT_", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(tc.KafkaHosts) > 0 {
|
||||
if !reflect.DeepEqual(tc.KafkaHosts, m.KafkaBootstrapServers) {
|
||||
t.Fatalf("--kafka-hosts expected: %v got: %v", tc.KafkaHosts, m.KafkaBootstrapServers)
|
||||
}
|
||||
}
|
||||
|
||||
if tc.RegistryURL != m.SchemaRegistryURL {
|
||||
t.Fatalf("--registry-url expected: %v got: %v", tc.RegistryURL, m.SchemaRegistryURL)
|
||||
}
|
||||
|
||||
if tc.Topic != m.Topic {
|
||||
t.Fatalf("--topic expected: %v got: %v", tc.Topic, m.Topic)
|
||||
}
|
||||
|
||||
if tc.Subject != m.Subject {
|
||||
t.Fatalf("--subject expected: %v got: %v", tc.Subject, m.Subject)
|
||||
}
|
||||
|
||||
if tc.SchemaFile != m.SchemaFile {
|
||||
t.Fatalf("--schema-file expected: %v got: %v", tc.SchemaFile, m.SchemaFile)
|
||||
}
|
||||
|
||||
if tc.Schema != m.Schema {
|
||||
t.Fatalf("--schema expected: %v got: %v", tc.Schema, m.Schema)
|
||||
}
|
||||
|
||||
if tc.Data != m.Data {
|
||||
t.Fatalf("--data expected: %v got: %v", tc.Data, m.Data)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
48
idk/cmd/molecula-consumer-csv/main.go
Normal file
48
idk/cmd/molecula-consumer-csv/main.go
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/csv"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
m := csv.NewMain()
|
||||
if err := pflag.LoadEnv(m, "IDKCSV_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
m.Rename()
|
||||
if m.DryRun {
|
||||
fields, err := m.ValidateHeaders()
|
||||
if err != nil {
|
||||
log.Printf("validation error: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("%+v\n", m)
|
||||
fmt.Printf("Parsed fields:\n")
|
||||
for _, f := range fields {
|
||||
fmt.Printf(" %-20T %+[1]v\n", f)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if m.Concurrency != 1 {
|
||||
m.Log().Infof("Concurrency is not supported for csv ingest. '--concurrency' flag will be ignored.")
|
||||
m.Concurrency = 1
|
||||
}
|
||||
|
||||
if err := m.Run(); err != nil {
|
||||
log := m.Log()
|
||||
if log == nil {
|
||||
// if we fail before a logger was instantiated
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
207
idk/cmd/molecula-consumer-csv/main_test.go
Normal file
207
idk/cmd/molecula-consumer-csv/main_test.go
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/jaffee/commandeer"
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/csv"
|
||||
pflag13 "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func TestConsumerCSVArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
|
||||
AssumeEmptyPilosa bool
|
||||
AutoGenerate bool
|
||||
BatchSize int
|
||||
Concurrency int
|
||||
IDField string
|
||||
PilosaHosts []string
|
||||
PilosaGRPCHosts []string
|
||||
Stats string
|
||||
Verbose bool
|
||||
ExpSplitBatchMode bool
|
||||
WriteCSV string
|
||||
Files []string
|
||||
Header []string
|
||||
Index string
|
||||
JustDoIt bool
|
||||
IgnoreHeader bool
|
||||
FutureRename bool
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{
|
||||
"", // os.Args[0] can be ignored
|
||||
},
|
||||
AssumeEmptyPilosa: false,
|
||||
AutoGenerate: false,
|
||||
BatchSize: 1,
|
||||
Concurrency: 1,
|
||||
IDField: "",
|
||||
PilosaHosts: []string{"localhost:10101"},
|
||||
PilosaGRPCHosts: []string{"localhost:20101"},
|
||||
Stats: "localhost:9093",
|
||||
Verbose: false,
|
||||
ExpSplitBatchMode: false,
|
||||
WriteCSV: "",
|
||||
Index: "",
|
||||
JustDoIt: false,
|
||||
IgnoreHeader: false,
|
||||
FutureRename: false,
|
||||
},
|
||||
{
|
||||
name: "long",
|
||||
args: []string{
|
||||
"molecula-consumer-csv",
|
||||
"--assume-empty-pilosa", "true",
|
||||
"--auto-generate", "true",
|
||||
"--batch-size", "12345",
|
||||
"--concurrency", "1",
|
||||
"--exp-split-batch-mode", "true",
|
||||
"--id-field", "id",
|
||||
"--index", "index_name",
|
||||
"--log-path", "/tmp/file.log",
|
||||
"--pack-bools", "true",
|
||||
"--pilosa-grpc-hosts", "grpc:1,grpc:2,grpc:3",
|
||||
"--pilosa-hosts", "pilosa:1,pilosa:2,pilosa:3",
|
||||
"--primary-key-fields", "primary_key",
|
||||
"--stats", "localhost:9093",
|
||||
"--tls.ca-certificate", "/tmp/file.ca",
|
||||
"--tls.certificate", "/tmp/file.certificate",
|
||||
"--tls.enable-client-verification", "true",
|
||||
"--tls.key", "/tmp/file.key",
|
||||
"--tls.skip-verify", "false",
|
||||
"--verbose", "true",
|
||||
"--write-csv", "/tmp/file.csv",
|
||||
"--files", "f1,f2,f3",
|
||||
"--header", "h1,h2,h3",
|
||||
"--index", "index-name",
|
||||
"--just-do-it", "true",
|
||||
"--ignore-header", "true",
|
||||
},
|
||||
AssumeEmptyPilosa: true,
|
||||
AutoGenerate: true,
|
||||
BatchSize: 12345,
|
||||
Concurrency: 1,
|
||||
IDField: "id",
|
||||
PilosaHosts: []string{"pilosa:1", "pilosa:2", "pilosa:3"},
|
||||
PilosaGRPCHosts: []string{"grpc:1", "grpc:2", "grpc:3"},
|
||||
Stats: "localhost:9093",
|
||||
Verbose: true,
|
||||
ExpSplitBatchMode: true,
|
||||
WriteCSV: "/tmp/file.csv",
|
||||
Files: []string{"f1", "f2", "f3"},
|
||||
Header: []string{"h1", "h2", "h3"},
|
||||
Index: "index-name",
|
||||
JustDoIt: true,
|
||||
IgnoreHeader: true,
|
||||
FutureRename: false,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{
|
||||
"molecula-consumer-csv",
|
||||
"--future.rename", "true",
|
||||
"--featurebase-hosts", "localhost:50101",
|
||||
"--featurebase-grpc-hosts", "localhost:60101",
|
||||
},
|
||||
AssumeEmptyPilosa: false,
|
||||
AutoGenerate: false,
|
||||
BatchSize: 1,
|
||||
Concurrency: 1,
|
||||
IDField: "",
|
||||
PilosaHosts: []string{"localhost:50101"},
|
||||
PilosaGRPCHosts: []string{"localhost:60101"},
|
||||
Stats: "localhost:9093",
|
||||
Verbose: false,
|
||||
ExpSplitBatchMode: false,
|
||||
WriteCSV: "",
|
||||
Index: "",
|
||||
JustDoIt: false,
|
||||
IgnoreHeader: false,
|
||||
FutureRename: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs := &pflag.FlagSet{FlagSet: pflag13.NewFlagSet(tc.args[0], pflag13.ExitOnError)}
|
||||
m := csv.NewMain()
|
||||
|
||||
if err := commandeer.LoadArgsEnv(fs, m, tc.args[1:], "IDKCSV_", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
m.Rename()
|
||||
|
||||
if tc.AssumeEmptyPilosa != m.AssumeEmptyPilosa {
|
||||
t.Fatalf("--assume-empty-pilosa expected: %v got: %v", tc.AssumeEmptyPilosa, m.AssumeEmptyPilosa)
|
||||
}
|
||||
if tc.AutoGenerate != m.AutoGenerate {
|
||||
t.Fatalf("--auto-generate expected: %v got: %v", tc.AutoGenerate, m.AutoGenerate)
|
||||
}
|
||||
if tc.BatchSize != m.BatchSize {
|
||||
t.Fatalf("--batch-size expected: %v got: %v", tc.BatchSize, m.BatchSize)
|
||||
}
|
||||
if tc.Concurrency != m.Concurrency {
|
||||
t.Fatalf("--concurrency expected: %v got: %v", tc.Concurrency, m.Concurrency)
|
||||
}
|
||||
if tc.IDField != m.IDField {
|
||||
t.Fatalf("--id-field expected: %v got: %v", tc.IDField, m.IDField)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PilosaHosts, m.PilosaHosts) {
|
||||
t.Fatalf("--pilosa-hosts expected: %v got: %v", tc.PilosaHosts, m.PilosaHosts)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PilosaGRPCHosts, m.PilosaGRPCHosts) {
|
||||
t.Fatalf("--pilosa-grpc-hosts expected: %v got: %v", tc.PilosaGRPCHosts, m.PilosaGRPCHosts)
|
||||
}
|
||||
if tc.Stats != m.Stats {
|
||||
t.Fatalf("--stats expected: %v got: %v", tc.Stats, m.Stats)
|
||||
}
|
||||
if tc.Verbose != m.Verbose {
|
||||
t.Fatalf("--verbose expected: %v got: %v", tc.Verbose, m.Verbose)
|
||||
}
|
||||
if tc.ExpSplitBatchMode != m.ExpSplitBatchMode {
|
||||
t.Fatalf("--exp-split-batch-mode expected: %v got: %v", tc.ExpSplitBatchMode, m.ExpSplitBatchMode)
|
||||
}
|
||||
if tc.WriteCSV != m.WriteCSV {
|
||||
t.Fatalf("--write-csv expected: %v got: %v", tc.WriteCSV, m.WriteCSV)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.Files, m.Files) {
|
||||
t.Fatalf("--files expected: %+v got: %+v", tc.Files, m.Files)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.Header, m.Header) {
|
||||
t.Fatalf("--header expected: %v got: %v", tc.Header, m.Header)
|
||||
}
|
||||
if tc.Index != m.Index {
|
||||
t.Fatalf("--index expected: %v got: %v", tc.Index, m.Index)
|
||||
}
|
||||
if tc.JustDoIt != m.JustDoIt {
|
||||
t.Fatalf("--just-do-it expected: %v got: %v", tc.JustDoIt, m.JustDoIt)
|
||||
}
|
||||
if tc.IgnoreHeader != m.IgnoreHeader {
|
||||
t.Fatalf("--ignore-header expected: %v got: %v", tc.IgnoreHeader, m.IgnoreHeader)
|
||||
}
|
||||
if tc.FutureRename != m.Future.Rename {
|
||||
t.Fatalf("--future.rename expected: %v got: %v", tc.FutureRename, m.Future.Rename)
|
||||
}
|
||||
|
||||
source, err := m.NewSource()
|
||||
// check for err as empty testcase fails to create Source
|
||||
if err == nil {
|
||||
csvSource := source.(*csv.Source)
|
||||
if csvSource.JustDoIt != m.JustDoIt {
|
||||
t.Fatalf("Source.JustDoIt does not match Main.JustDoIt")
|
||||
}
|
||||
if csvSource.IgnoreHeader != m.IgnoreHeader {
|
||||
t.Fatalf("Source.IgnoreHeader does not match Main.IgnoreHeader")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
4
idk/cmd/molecula-consumer-github/README.md
Normal file
4
idk/cmd/molecula-consumer-github/README.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
### Example
|
||||
```
|
||||
./molecula-consumer-github -d id -i events --record-type event --batch-size=100000 --start-time "2015-10-13T00:00:00Z"
|
||||
```
|
||||
435
idk/cmd/molecula-consumer-github/main.go
Normal file
435
idk/cmd/molecula-consumer-github/main.go
Normal file
|
|
@ -0,0 +1,435 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"expvar"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
MinWaitDuration = 5 * time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
ghEventsVar = expvar.NewInt("gh_events")
|
||||
ghEventsPerSecVar = expvar.NewFloat("gh_events_per_sec")
|
||||
)
|
||||
|
||||
func main() {
|
||||
m := NewMain()
|
||||
if err := pflag.LoadEnv(m, "IDKGITHUB_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
m.Rename()
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
if err := m.Run(); err != nil {
|
||||
log := m.Log()
|
||||
if log == nil {
|
||||
// if we fail before a logger was instantiated
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
type Main struct {
|
||||
idk.Main `flag:"!embed"`
|
||||
RecordType string `help:"Output record type"`
|
||||
StartTime time.Time `help:"Start time"`
|
||||
EndTime time.Time `help:"End time, optional"`
|
||||
CacheDir string `help:"Directory for local data files"`
|
||||
}
|
||||
|
||||
func NewMain() *Main {
|
||||
m := &Main{
|
||||
Main: *idk.NewMain(),
|
||||
StartTime: time.Now().UTC().Truncate(24 * time.Hour),
|
||||
}
|
||||
m.Main.Namespace = "ingester_github"
|
||||
|
||||
// Start streaming events to channel in timestamp order.
|
||||
|
||||
m.NewSource = func() (idk.Source, error) {
|
||||
if !IsValidRecordType(m.RecordType) {
|
||||
return nil, errors.New("invalid record type; must be 'event', 'user', 'issue' or 'repo'")
|
||||
}
|
||||
events := make(chan Event)
|
||||
go m.stream(events)
|
||||
source := NewSource(events)
|
||||
source.RecordType = m.RecordType
|
||||
source.Log = m.Main.Log()
|
||||
// source.JustDoIt = m.JustDoIt
|
||||
return source, nil
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *Main) Run() error {
|
||||
// Set up HTTP endpoint to provide /debug/vars & /debug/pprof
|
||||
fmt.Println("Serving debug endpoint at http://localhost:7070/debug")
|
||||
go func() { _ = http.ListenAndServe(":7070", nil) }()
|
||||
|
||||
go m.monitor()
|
||||
|
||||
return m.Main.Run()
|
||||
}
|
||||
|
||||
func (m *Main) stream(events chan Event) {
|
||||
defer close(events)
|
||||
|
||||
// Ensure start time is aligned to hour.
|
||||
t := m.StartTime.Truncate(1 * time.Hour)
|
||||
|
||||
for {
|
||||
// Stop once we reach the end time.
|
||||
if !m.EndTime.IsZero() && t.After(m.EndTime) {
|
||||
m.Main.Log().Printf("end of time range, stopping")
|
||||
break
|
||||
}
|
||||
|
||||
// Fetch JSON data and convert to CSV.
|
||||
// If the data doesn't exist, wait and retry.
|
||||
if err := m.process(events, t); err == errNotFound {
|
||||
d := time.Until(t)
|
||||
if d < MinWaitDuration {
|
||||
d = MinWaitDuration
|
||||
}
|
||||
m.Main.Log().Printf("cannot fetch data for %s, waiting %s", t.Format(time.RFC3339), d)
|
||||
time.Sleep(d)
|
||||
continue
|
||||
} else if err != nil {
|
||||
m.Main.Log().Printf("error processing data: %s", err)
|
||||
events <- Event{err: err}
|
||||
return
|
||||
}
|
||||
|
||||
// Move to the next timestamp.
|
||||
t = t.Add(time.Hour)
|
||||
}
|
||||
}
|
||||
|
||||
// process fetches the GitHub Archive gzipped JSON data streams each event to the events channel.
|
||||
func (m *Main) process(events chan Event, t time.Time) error {
|
||||
log.Printf("processing %s", t.Format(time.RFC3339))
|
||||
defer log.Printf("done processing %s", t.Format(time.RFC3339))
|
||||
|
||||
// Fetch a reader for the timestamp.
|
||||
r, err := m.openURLReader(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
// Decompress stream.
|
||||
gr, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gr.Close()
|
||||
|
||||
// Read data as JSON.
|
||||
dec := json.NewDecoder(gr)
|
||||
counter := 0
|
||||
for {
|
||||
// Unmarshal single event from JSON stream.
|
||||
var event Event
|
||||
if err := dec.Decode(&event); err == io.EOF {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if counter%20000 == 0 {
|
||||
m.Main.Log().Printf("record: id=%d", event.ID)
|
||||
}
|
||||
counter++
|
||||
|
||||
// Send event to channel.
|
||||
events <- event
|
||||
ghEventsVar.Add(1) // total event count
|
||||
}
|
||||
}
|
||||
|
||||
// monitor runs in a separate goroutine and updates the ingestion rate stats.
|
||||
func (m *Main) monitor() {
|
||||
ticker := time.NewTicker(1 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
var lastTime time.Time
|
||||
var lastN int64
|
||||
for range ticker.C {
|
||||
now, n := time.Now(), ghEventsVar.Value()
|
||||
if !lastTime.IsZero() {
|
||||
ghEventsPerSecVar.Set(float64(lastN-n) / lastTime.Sub(now).Seconds())
|
||||
}
|
||||
lastTime, lastN = now, n
|
||||
}
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
RecordType string
|
||||
Log logger.Logger
|
||||
|
||||
events chan Event
|
||||
}
|
||||
|
||||
func NewSource(events chan Event) *Source {
|
||||
return &Source{events: events}
|
||||
}
|
||||
|
||||
func (s *Source) Record() (idk.Record, error) {
|
||||
for {
|
||||
event, ok := <-s.events
|
||||
if !ok {
|
||||
return nil, io.EOF
|
||||
}
|
||||
// Break out the error to hand back to IDK.
|
||||
if event.err != nil {
|
||||
return nil, event.err
|
||||
}
|
||||
|
||||
var record idk.Record
|
||||
switch s.RecordType {
|
||||
case RecordTypeEvent:
|
||||
record = EventRecord(event)
|
||||
case RecordTypeUser:
|
||||
record = UserRecord(event)
|
||||
case RecordTypeRepo:
|
||||
record = RepoRecord(event)
|
||||
case RecordTypeIssue:
|
||||
record = IssueRecord(event)
|
||||
default:
|
||||
panic(fmt.Errorf("invalid source record type: %q", s.RecordType))
|
||||
}
|
||||
|
||||
// If record implementation a validation function then check it.
|
||||
// Retry the next record if the record is invalid.
|
||||
if record, ok := record.(interface {
|
||||
Valid() bool
|
||||
}); ok && !record.Valid() {
|
||||
continue
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) Schema() []idk.Field {
|
||||
switch s.RecordType {
|
||||
case RecordTypeEvent:
|
||||
return s.eventSchema()
|
||||
case RecordTypeUser:
|
||||
return s.userSchema()
|
||||
case RecordTypeRepo:
|
||||
return s.repoSchema()
|
||||
case RecordTypeIssue:
|
||||
return s.issueSchema()
|
||||
default:
|
||||
panic(fmt.Sprintf("no schema for record type: %q", s.RecordType))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) eventSchema() []idk.Field {
|
||||
idMin := int64(0)
|
||||
idMax := int64(1 << 31)
|
||||
return []idk.Field{
|
||||
idk.IDField{NameVal: "id"},
|
||||
idk.StringField{NameVal: "type"},
|
||||
idk.IDField{NameVal: "actor_id", Quantum: "YMDH"},
|
||||
idk.IDField{NameVal: "repo_id", Quantum: "YMDH"},
|
||||
idk.IntField{NameVal: "actor_bsi", Min: &idMin, Max: &idMax},
|
||||
idk.IntField{NameVal: "repo_bsi", Min: &idMin, Max: &idMax},
|
||||
idk.RecordTimeField{NameVal: "created_at", Layout: time.RFC3339},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) userSchema() []idk.Field {
|
||||
return []idk.Field{
|
||||
idk.StringField{NameVal: "login"},
|
||||
idk.IDField{NameVal: "github_id"},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) repoSchema() []idk.Field {
|
||||
return []idk.Field{
|
||||
idk.StringField{NameVal: "name"},
|
||||
idk.IDField{NameVal: "github_id"},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) issueSchema() []idk.Field {
|
||||
idMin := int64(0)
|
||||
idMax := int64(1 << 31)
|
||||
return []idk.Field{
|
||||
idk.IDField{NameVal: "id"},
|
||||
idk.StringField{NameVal: "url"},
|
||||
idk.IntField{NameVal: "number", Min: &idMin, Max: &idMax},
|
||||
idk.IDField{NameVal: "comments", Mutex: true},
|
||||
idk.RecordTimeField{NameVal: "created_at", Layout: time.RFC3339},
|
||||
}
|
||||
}
|
||||
func (s *Source) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Output type constants.
|
||||
const (
|
||||
RecordTypeEvent = "event"
|
||||
RecordTypeUser = "user"
|
||||
RecordTypeRepo = "repo"
|
||||
RecordTypeIssue = "issue"
|
||||
)
|
||||
|
||||
// IsValidRecordType returns true if s is a valid record type.
|
||||
func IsValidRecordType(s string) bool {
|
||||
switch s {
|
||||
case RecordTypeEvent, RecordTypeUser, RecordTypeRepo, RecordTypeIssue:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type EventRecord Event
|
||||
|
||||
func (r EventRecord) Data() []interface{} {
|
||||
return []interface{}{r.ID, r.Type, r.Actor.ID, r.Repo.ID, r.Actor.ID, r.Repo.ID, r.CreatedAt}
|
||||
}
|
||||
|
||||
func (r EventRecord) Commit(ctx context.Context) error { return nil }
|
||||
|
||||
type UserRecord Event
|
||||
|
||||
func (r UserRecord) Data() []interface{} {
|
||||
return []interface{}{r.Actor.Login, r.Actor.ID}
|
||||
}
|
||||
|
||||
func (r UserRecord) Commit(ctx context.Context) error { return nil }
|
||||
|
||||
type RepoRecord Event
|
||||
|
||||
func (r RepoRecord) Data() []interface{} {
|
||||
return []interface{}{r.Repo.Name, r.Repo.ID}
|
||||
}
|
||||
|
||||
func (r RepoRecord) Commit(ctx context.Context) error { return nil }
|
||||
|
||||
type IssueRecord Event
|
||||
|
||||
func (r IssueRecord) Valid() bool {
|
||||
return r.Type == "IssuesEvent" || r.Type == "IssueCommentEvent"
|
||||
}
|
||||
|
||||
func (r IssueRecord) Data() []interface{} {
|
||||
var issue Issue
|
||||
switch r.Type {
|
||||
case "IssuesEvent":
|
||||
var payload IssuePayload
|
||||
_ = json.Unmarshal(r.Payload, &payload)
|
||||
issue = payload.Issue
|
||||
case "IssueCommentEvent":
|
||||
var payload IssueCommentPayload
|
||||
_ = json.Unmarshal(r.Payload, &payload)
|
||||
issue = payload.Issue
|
||||
}
|
||||
return []interface{}{issue.ID, issue.URL, issue.Number, issue.Comments, r.CreatedAt}
|
||||
}
|
||||
|
||||
func (r IssueRecord) Commit(ctx context.Context) error { return nil }
|
||||
|
||||
func (m *Main) openURLReader(t time.Time) (io.ReadCloser, error) {
|
||||
filename := fmt.Sprintf("%04d-%02d-%02d-%d.json.gz", t.Year(), t.Month(), t.Day(), t.Hour())
|
||||
cachePath := filepath.Join(m.CacheDir, filename)
|
||||
|
||||
if m.CacheDir != "" {
|
||||
if fi, err := os.Stat(cachePath); err == nil && fi.Size() != 0 {
|
||||
m.Log().Printf("using cached file")
|
||||
return os.Open(cachePath)
|
||||
}
|
||||
}
|
||||
|
||||
rawurl := "https://data.gharchive.org/" + filename
|
||||
resp, err := http.Get(rawurl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if resp.StatusCode == http.StatusNotFound {
|
||||
resp.Body.Close()
|
||||
return nil, errNotFound
|
||||
} else if resp.StatusCode >= 400 {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("invalid status code: code=%d url=%s", resp.StatusCode, rawurl)
|
||||
}
|
||||
|
||||
// If no cache directory, return body directly.
|
||||
if m.CacheDir == "" {
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// If cache enabled, write to file first and then return.
|
||||
if buf, err := ioutil.ReadAll(resp.Body); err != nil {
|
||||
return nil, err
|
||||
} else if err := ioutil.WriteFile(cachePath+".tmp", buf, 0666); err != nil {
|
||||
return nil, err
|
||||
} else if err := os.Rename(cachePath+".tmp", cachePath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Log().Printf("cached file written")
|
||||
return os.Open(cachePath)
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
ID int `json:"id,string"`
|
||||
Type string `json:"type"`
|
||||
Actor *Actor `json:"actor"`
|
||||
Repo *Repo `json:"repo"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
|
||||
err error
|
||||
}
|
||||
|
||||
type Actor struct {
|
||||
ID int `json:"id"`
|
||||
Login string `json:"login"`
|
||||
}
|
||||
|
||||
type Repo struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type Issue struct {
|
||||
ID int `json:"id"`
|
||||
URL string `json:"url"`
|
||||
Number int `json:"number"`
|
||||
Comments int `json:"comments"`
|
||||
}
|
||||
|
||||
type IssuePayload struct {
|
||||
Issue Issue `json:"issue"`
|
||||
}
|
||||
|
||||
type IssueCommentPayload struct {
|
||||
Issue Issue `json:"issue"`
|
||||
}
|
||||
|
||||
// errNotFound is returned when a data file does not exist.
|
||||
var errNotFound = errors.New("not found")
|
||||
32
idk/cmd/molecula-consumer-kafka-delete/main.go
Normal file
32
idk/cmd/molecula-consumer-kafka-delete/main.go
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka"
|
||||
)
|
||||
|
||||
func main() {
|
||||
dm, err := kafka.NewMain()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
dm.Delete = true
|
||||
|
||||
if err := pflag.LoadEnv(dm, "CONSUMER_DEL_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
dm.Rename()
|
||||
if dm.DryRun {
|
||||
log.Printf("%+v\n", dm)
|
||||
return
|
||||
}
|
||||
if err := dm.Run(); err != nil {
|
||||
logger := dm.Log()
|
||||
if logger != nil {
|
||||
logger.Printf("Error running command: %v", err)
|
||||
}
|
||||
log.Fatal(err) // make sure we log to stderr if the error happened before logging was set up
|
||||
}
|
||||
}
|
||||
192
idk/cmd/molecula-consumer-kafka-delete/main_test.go
Normal file
192
idk/cmd/molecula-consumer-kafka-delete/main_test.go
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jaffee/commandeer"
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka"
|
||||
pflag13 "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func TestConsumerKafkaDeleteArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
|
||||
AssumeEmptyPilosa bool
|
||||
AutoGenerate bool
|
||||
BatchSize int
|
||||
Concurrency int
|
||||
IDField string
|
||||
PilosaHosts []string
|
||||
PilosaGRPCHosts []string
|
||||
Stats string
|
||||
Verbose bool
|
||||
ExpSplitBatchMode bool
|
||||
WriteCSV string
|
||||
Index string
|
||||
PrimaryKeyFields []string
|
||||
Pprof string
|
||||
KafkaHosts []string
|
||||
RegistryURL string
|
||||
MaxMsgs uint64
|
||||
Group string
|
||||
Topics []string
|
||||
Timeout time.Duration
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{
|
||||
"", // os.Args[0] can be ignored
|
||||
},
|
||||
AssumeEmptyPilosa: false,
|
||||
AutoGenerate: false,
|
||||
BatchSize: 1,
|
||||
Concurrency: 1,
|
||||
IDField: "",
|
||||
PilosaHosts: []string{"localhost:10101"},
|
||||
PilosaGRPCHosts: []string{"localhost:20101"},
|
||||
Stats: "localhost:9093",
|
||||
Verbose: false,
|
||||
ExpSplitBatchMode: false,
|
||||
WriteCSV: "",
|
||||
Index: "",
|
||||
Pprof: "localhost:6062",
|
||||
KafkaHosts: []string{"localhost:9092"},
|
||||
RegistryURL: "http://localhost:8081",
|
||||
MaxMsgs: uint64(0),
|
||||
Group: "defaultgroup",
|
||||
Topics: []string{"defaulttopic"},
|
||||
Timeout: time.Second,
|
||||
},
|
||||
{
|
||||
name: "long",
|
||||
args: []string{
|
||||
"molecula-consumer-kafka-delete",
|
||||
"--assume-empty-pilosa", "true",
|
||||
"--auto-generate", "true",
|
||||
"--batch-size", "12345",
|
||||
"--concurrency", "1",
|
||||
"--exp-split-batch-mode", "true",
|
||||
"--id-field", "id",
|
||||
"--index", "index_name",
|
||||
"--log-path", "/tmp/file.log",
|
||||
"--pack-bools", "true",
|
||||
"--pilosa-grpc-hosts", "grpc:1,grpc:2,grpc:3",
|
||||
"--pilosa-hosts", "pilosa:1,pilosa:2,pilosa:3",
|
||||
"--stats", "localhost:9093",
|
||||
"--tls.ca-certificate", "/tmp/file.ca",
|
||||
"--tls.certificate", "/tmp/file.certificate",
|
||||
"--tls.enable-client-verification", "true",
|
||||
"--tls.key", "/tmp/file.key",
|
||||
"--tls.skip-verify", "false",
|
||||
"--verbose", "true",
|
||||
"--write-csv", "/tmp/file.csv",
|
||||
"--index", "index-name",
|
||||
"--primary-key-fields", "k1",
|
||||
"--pprof", "localhost:6666",
|
||||
"--kafka-bootstrap-servers", "kafka:1,kafka:2,kafka:3",
|
||||
"--schema-registry-url", "registry.molecula.com",
|
||||
"--max-msgs", "1234",
|
||||
"--group", "molecula",
|
||||
"--topics", "t1,t2,t3,t4",
|
||||
"--timeout", "123ms",
|
||||
},
|
||||
AssumeEmptyPilosa: true,
|
||||
AutoGenerate: true,
|
||||
BatchSize: 12345,
|
||||
Concurrency: 1,
|
||||
IDField: "id",
|
||||
PilosaHosts: []string{"pilosa:1", "pilosa:2", "pilosa:3"},
|
||||
PilosaGRPCHosts: []string{"grpc:1", "grpc:2", "grpc:3"},
|
||||
Stats: "localhost:9093",
|
||||
Verbose: true,
|
||||
ExpSplitBatchMode: true,
|
||||
WriteCSV: "/tmp/file.csv",
|
||||
Index: "index-name",
|
||||
PrimaryKeyFields: []string{"k1"},
|
||||
KafkaHosts: []string{"kafka:1", "kafka:2", "kafka:3"},
|
||||
RegistryURL: "registry.molecula.com",
|
||||
MaxMsgs: uint64(1234),
|
||||
Group: "molecula",
|
||||
Topics: []string{"t1", "t2", "t3", "t4"},
|
||||
Timeout: 123 * time.Millisecond,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs := &pflag.FlagSet{FlagSet: pflag13.NewFlagSet(tc.args[0], pflag13.ExitOnError)}
|
||||
m, err := kafka.NewMain()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Delete = true
|
||||
|
||||
if err := commandeer.LoadArgsEnv(fs, m, tc.args[1:], "CONSUMER_DEL_", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if tc.AssumeEmptyPilosa != m.AssumeEmptyPilosa {
|
||||
t.Fatalf("--assume-empty-pilosa expected: %v got: %v", tc.AssumeEmptyPilosa, m.AssumeEmptyPilosa)
|
||||
}
|
||||
if tc.AutoGenerate != m.AutoGenerate {
|
||||
t.Fatalf("--auto-generate expected: %v got: %v", tc.AutoGenerate, m.AutoGenerate)
|
||||
}
|
||||
if tc.BatchSize != m.BatchSize {
|
||||
t.Fatalf("--batch-size expected: %v got: %v", tc.BatchSize, m.BatchSize)
|
||||
}
|
||||
if tc.Concurrency != m.Concurrency {
|
||||
t.Fatalf("--concurrency expected: %v got: %v", tc.Concurrency, m.Concurrency)
|
||||
}
|
||||
if tc.IDField != m.IDField {
|
||||
t.Fatalf("--id-field expected: %v got: %v", tc.IDField, m.IDField)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PilosaHosts, m.PilosaHosts) {
|
||||
t.Fatalf("--pilosa-hosts expected: %v got: %v", tc.PilosaHosts, m.PilosaHosts)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PilosaGRPCHosts, m.PilosaGRPCHosts) {
|
||||
t.Fatalf("--pilosa-grpc-hosts expected: %v got: %v", tc.PilosaGRPCHosts, m.PilosaGRPCHosts)
|
||||
}
|
||||
if tc.Stats != m.Stats {
|
||||
t.Fatalf("--stats expected: %v got: %v", tc.Stats, m.Stats)
|
||||
}
|
||||
if tc.Verbose != m.Verbose {
|
||||
t.Fatalf("--verbose expected: %v got: %v", tc.Verbose, m.Verbose)
|
||||
}
|
||||
if tc.ExpSplitBatchMode != m.ExpSplitBatchMode {
|
||||
t.Fatalf("--exp-split-batch-mode expected: %v got: %v", tc.ExpSplitBatchMode, m.ExpSplitBatchMode)
|
||||
}
|
||||
if tc.WriteCSV != m.WriteCSV {
|
||||
t.Fatalf("--write-csv expected: %v got: %v", tc.WriteCSV, m.WriteCSV)
|
||||
}
|
||||
if tc.Index != m.Index {
|
||||
t.Fatalf("--index expected: %v got: %v", tc.Index, m.Index)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PrimaryKeyFields, m.PrimaryKeyFields) {
|
||||
t.Fatalf("--primary-key-fields expected: %v got: %v", tc.PrimaryKeyFields, m.PrimaryKeyFields)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.KafkaHosts, m.KafkaBootstrapServers) {
|
||||
t.Fatalf("--kafka-hosts expected: %v got: %v", tc.KafkaHosts, m.KafkaBootstrapServers)
|
||||
}
|
||||
if tc.RegistryURL != m.SchemaRegistryURL {
|
||||
t.Fatalf("--registry-url expected: %v got: %v", tc.RegistryURL, m.SchemaRegistryURL)
|
||||
}
|
||||
if tc.MaxMsgs != m.MaxMsgs {
|
||||
t.Fatalf("--max-msgs expected: %v got: %v", tc.MaxMsgs, m.MaxMsgs)
|
||||
}
|
||||
if tc.Group != m.Group {
|
||||
t.Fatalf("--group expected: %v got: %v", tc.Group, m.Group)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.Topics, m.Topics) {
|
||||
t.Fatalf("--topics expected: %v got: %v", tc.Topics, m.Topics)
|
||||
}
|
||||
if tc.Timeout != m.Timeout {
|
||||
t.Fatalf("--timeout expected: %v got: %v", tc.Timeout, m.Timeout)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
36
idk/cmd/molecula-consumer-kafka-sasl/main.go
Normal file
36
idk/cmd/molecula-consumer-kafka-sasl/main.go
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka_sasl"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
m, err := kafka_sasl.NewMain()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := pflag.LoadEnv(m, "CONSUMER_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
m.Rename()
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
|
||||
if err := m.Run(); err != nil {
|
||||
log := m.Log()
|
||||
if log == nil {
|
||||
// if we fail before a logger was instantiated
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
67
idk/cmd/molecula-consumer-kafka-sasl/readme.md
Normal file
67
idk/cmd/molecula-consumer-kafka-sasl/readme.md
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
## Molecula Consumer Kafka SASL
|
||||
|
||||
#### Background
|
||||
|
||||
The goal of **Molecula Consumer Kafka SASL** is to allow IDK to connect to Kafka brokers with SASL-SSL enabled. Currently, the Kafka library we are using in **Molecula Consumer Kafka** and **Molecula Consumer Kafka Static** is Kafka-go from [Segment IO](https://github.com/segmentio/kafka-go) and it does not seem to support both SASL and SSL. After some research, we found that [Confluent Kafka Go](https://github.com/segmentio/kafka-go) is a tool we can use to connect to Kafka brokers with SASL and SSL. **Molecula Consumer Kafka SASL** was written based on **Molecula Consumer Kafka Static**.
|
||||
|
||||
#### Future
|
||||
|
||||
There should be a ticket coming soon to re-write **Molecula Consumer Kafka** with [Confluent Kafka Go](https://github.com/segmentio/kafka-go). At this moment, one of the challenges that will come up is finding a way to cross compile cgo through different architectures (**arm64!**) [*make docker-release*]. Once **Molecula Consumer Kafka** re-write with [Confluent Kafka Go](https://github.com/segmentio/kafka-go) is done, Kafka-go from [Segment IO](https://github.com/segmentio/kafka-go) should be removed as there is no need to have 2 libraries that basically do the same thing to exist within the project.
|
||||
|
||||
#### Kafka SASL
|
||||
|
||||
**Molecula Consumer Kafka SASL** should function and behave just like **Molecula Consumer Kafka Static**. The only difference is: **Molecula Consumer Kafka SASL** will be able to connect to Kafka brokers with SASL and SSL enabled by providing the needed information.
|
||||
|
||||
#### New options
|
||||
|
||||
The following options were added for Kafka SASL. None of the newly added are required.
|
||||
- Username: SASL authentication username
|
||||
- Password: SASL authentication password
|
||||
- IdAlgorithm: The endpoint identification algorithm used by clients to validate server host name. The default value is https.
|
||||
- SaslMechanism: SASL mechanism to use for authentication. Range: GSSAPI, PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER
|
||||
- SecurityProtocol: Protocol used to communicate with brokers. Range: plaintext, ssl, sasl_plaintext, sasl_ssl
|
||||
- SslCaLocation: File or directory path to CA certificate(s) for verifying the broker's key.
|
||||
- SslCertificateLocation: Path to client's public key (PEM) used for authentication.
|
||||
- SslKeyLocation: Path to client's private key (PEM) used for authentication.
|
||||
- SslKeyPassword: Private key passphrase (for use with ssl.key.location and set_ssl_cert()).
|
||||
|
||||
#### How to start Kafka SASL and connect to Lccal IDK Broker with SASL and SSL enabled
|
||||
|
||||
1. Start Local IDK by navigating to project IDK then enter: ***make shutdown && sudo make startup***
|
||||
- Looking at **docker-compose.yml**, non SASL & non SSL broker is at kafka:9092
|
||||
- SASL & SSL broker is at kafka:9094
|
||||
2. Connect to kafka:9094
|
||||
- The SASL and SSL information needed to connect to kafka:9094 is under docker-sasl/client-ssl-test.conf
|
||||
- test connection by listing the topics in kafka:9094:
|
||||
- kafka-topics --bootstrap-server kafka:9094 --list **--command-config client-ssl-test.conf.conf**
|
||||
3. Create a new topic to test sending/receiving messages
|
||||
- kafka-topics --bootstrap-server kafka:9094 --create --replication-factor 1 --partitions 10 **--topic testSasl** --command-config client-ssl-test.conf
|
||||
4. Create a header file
|
||||
- Since **Molecula Consumer Kafka SASL** was written based on **Molecula Consumer Kafka Static**, it needs a header file that has a schema inside it
|
||||
- Example header.json:
|
||||
```
|
||||
[
|
||||
{
|
||||
"name":"test_string",
|
||||
"path":[
|
||||
"s"
|
||||
],
|
||||
"type":"string"
|
||||
}
|
||||
]
|
||||
```
|
||||
- example message that matches the above schema: **{"s":"test 1"}**
|
||||
5. Create a producer
|
||||
- kafka-console-producer --bootstrap-server kafka:9094 **--topic testSasl** --producer.config client-ssl-test.conf
|
||||
6. SSL Certificates:
|
||||
- There is a script to generate SSL certificates under docker-sasl/ssl_keys/gen-ssl-certs.sh
|
||||
- Can also use existing certs in the above folder and feed cert files to molecula-consumer-kafka-sasl
|
||||
- These certs are needed since in this scenario, we are using kafka with SSL enabled on localhost so we have to geenrate self-signed SSL certs.
|
||||
7. Run molecula-consumer-kafka-sasl
|
||||
- If binary is not there, build by using **go build**
|
||||
- Run :
|
||||
- ./molecula-consumer-kafka-sasl --auto-generate --index test_string --group testGroup --header header.json --topics testSasl --kafka-hosts kafka:9094 --kafka-sasl.username kafkaClient1 --kafka-sasl.password kafkaClient1pw --kafka-sasl.sasl-mechanism PLAIN --kafka-sasl.security-protocol SASL_SSL --kafka-sasl.ssl-ca-location ../../docker-sasl/ssl_keys/ca-cert --kafka-sasl.ssl-certificate-location ../../docker-sasl/ssl_keys/client_kafkaClient_client.pem --kafka-sasl.ssl-key-location ../../docker-sasl/ssl_keys/client_kafkaClient_client.key --kafka-sasl.ssl-key-password 123456
|
||||
8. Send a message with kafka-console-producer
|
||||
9. molecula-consumer-kafka-sasl should be able to receive the message and commit it to locally
|
||||
10. Double check by going to local pilosa (http://127.0.0.1:10101/) -> query builder -> select test_string in the Table dropdown -> Run
|
||||
- the results should show the messages that were sent by kafka-console-producer
|
||||
32
idk/cmd/molecula-consumer-kafka-static/main.go
Normal file
32
idk/cmd/molecula-consumer-kafka-static/main.go
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka_static"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
m := kafka_static.NewMain()
|
||||
if err := pflag.LoadEnv(m, "CONSUMER_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
m.Rename()
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
if err := m.Run(); err != nil {
|
||||
log := m.Log()
|
||||
if log == nil {
|
||||
// if we fail before a logger was instantiated
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
37
idk/cmd/molecula-consumer-kafka/main.go
Normal file
37
idk/cmd/molecula-consumer-kafka/main.go
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
m, err := kafka.NewMain()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if err := pflag.LoadEnv(m, "CONSUMER_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
m.Rename()
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
|
||||
if err := m.Run(); err != nil {
|
||||
log := m.Log()
|
||||
if log == nil {
|
||||
// if we fail before a logger was instantiated
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
206
idk/cmd/molecula-consumer-kafka/main_test.go
Normal file
206
idk/cmd/molecula-consumer-kafka/main_test.go
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jaffee/commandeer"
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka"
|
||||
pflag13 "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func TestConsumerKafkaArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
|
||||
AssumeEmptyPilosa bool
|
||||
AutoGenerate bool
|
||||
ExternalGenerate bool
|
||||
OffsetMode bool
|
||||
BatchSize int
|
||||
Concurrency int
|
||||
IDField string
|
||||
PilosaHosts []string
|
||||
PilosaGRPCHosts []string
|
||||
Stats string
|
||||
Verbose bool
|
||||
ExpSplitBatchMode bool
|
||||
WriteCSV string
|
||||
Index string
|
||||
PrimaryKeyFields []string
|
||||
Pprof string
|
||||
KafkaHosts []string
|
||||
RegistryURL string
|
||||
ConfluentCommand idk.ConfluentCommand
|
||||
Group string
|
||||
Topics []string
|
||||
Timeout time.Duration
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{
|
||||
"", // os.Args[0] can be ignored
|
||||
},
|
||||
AssumeEmptyPilosa: false,
|
||||
AutoGenerate: false,
|
||||
ExternalGenerate: false,
|
||||
OffsetMode: true,
|
||||
BatchSize: 1,
|
||||
Concurrency: 1,
|
||||
IDField: "",
|
||||
PilosaHosts: []string{"localhost:10101"},
|
||||
PilosaGRPCHosts: []string{"localhost:20101"},
|
||||
Stats: "localhost:9093",
|
||||
Verbose: false,
|
||||
ExpSplitBatchMode: false,
|
||||
WriteCSV: "",
|
||||
Index: "",
|
||||
Pprof: "localhost:6062",
|
||||
KafkaHosts: []string{"localhost:9092"},
|
||||
RegistryURL: "http://localhost:8081",
|
||||
Group: "defaultgroup",
|
||||
Topics: []string{"defaulttopic"},
|
||||
Timeout: time.Second,
|
||||
},
|
||||
{
|
||||
name: "long",
|
||||
args: []string{
|
||||
"molecula-consumer-kafka",
|
||||
"--assume-empty-pilosa", "true",
|
||||
"--auto-generate", "true",
|
||||
"--external-generate",
|
||||
"--batch-size", "12345",
|
||||
"--concurrency", "1",
|
||||
"--exp-split-batch-mode", "true",
|
||||
"--id-field", "id",
|
||||
"--index", "index_name",
|
||||
"--kafka-debug", "consumer",
|
||||
"--log-path", "/tmp/file.log",
|
||||
"--pack-bools", "true",
|
||||
"--pilosa-grpc-hosts", "grpc:1,grpc:2,grpc:3",
|
||||
"--pilosa-hosts", "pilosa:1,pilosa:2,pilosa:3",
|
||||
"--stats", "localhost:9093",
|
||||
"--tls.ca-certificate", "/tmp/file.ca",
|
||||
"--tls.certificate", "/tmp/file.certificate",
|
||||
"--tls.enable-client-verification", "true",
|
||||
"--tls.key", "/tmp/file.key",
|
||||
"--tls.skip-verify", "false",
|
||||
"--verbose", "true",
|
||||
"--write-csv", "/tmp/file.csv",
|
||||
"--index", "index-name",
|
||||
"--primary-key-fields", "k1",
|
||||
"--pprof", "localhost:6666",
|
||||
"--kafka-bootstrap-servers", "kafka:1,kafka:2,kafka:3",
|
||||
"--schema-registry-url", "registry.molecula.com",
|
||||
"--group", "molecula",
|
||||
"--topics", "t1,t2,t3,t4",
|
||||
"--timeout", "123ms",
|
||||
},
|
||||
AssumeEmptyPilosa: true,
|
||||
ExternalGenerate: true,
|
||||
AutoGenerate: true,
|
||||
BatchSize: 12345,
|
||||
Concurrency: 1,
|
||||
IDField: "id",
|
||||
PilosaHosts: []string{"pilosa:1", "pilosa:2", "pilosa:3"},
|
||||
PilosaGRPCHosts: []string{"grpc:1", "grpc:2", "grpc:3"},
|
||||
Stats: "localhost:9093",
|
||||
Verbose: true,
|
||||
ExpSplitBatchMode: true,
|
||||
WriteCSV: "/tmp/file.csv",
|
||||
Index: "index-name",
|
||||
PrimaryKeyFields: []string{"k1"},
|
||||
KafkaHosts: []string{"kafka:1", "kafka:2", "kafka:3"},
|
||||
RegistryURL: "registry.molecula.com",
|
||||
ConfluentCommand: idk.ConfluentCommand{
|
||||
KafkaDebug: "consumer",
|
||||
},
|
||||
Group: "molecula",
|
||||
Topics: []string{"t1", "t2", "t3", "t4"},
|
||||
Timeout: 123 * time.Millisecond,
|
||||
OffsetMode: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs := &pflag.FlagSet{FlagSet: pflag13.NewFlagSet(tc.args[0], pflag13.ExitOnError)}
|
||||
m, err := kafka.NewMain()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := commandeer.LoadArgsEnv(fs, m, tc.args[1:], "CONSUMER_", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if tc.AssumeEmptyPilosa != m.AssumeEmptyPilosa {
|
||||
t.Fatalf("--assume-empty-pilosa expected: %v got: %v", tc.AssumeEmptyPilosa, m.AssumeEmptyPilosa)
|
||||
}
|
||||
if tc.AutoGenerate != m.AutoGenerate {
|
||||
t.Fatalf("--auto-generate expected: %v got: %v", tc.AutoGenerate, m.AutoGenerate)
|
||||
}
|
||||
if tc.ExternalGenerate != m.ExternalGenerate {
|
||||
t.Fatalf("--external-generate expected: %v got: %v", tc.ExternalGenerate, m.ExternalGenerate)
|
||||
}
|
||||
if tc.OffsetMode != m.OffsetMode {
|
||||
t.Fatalf("--offset-mode expected: %v got: %v", tc.OffsetMode, m.OffsetMode)
|
||||
}
|
||||
if tc.BatchSize != m.BatchSize {
|
||||
t.Fatalf("--batch-size expected: %v got: %v", tc.BatchSize, m.BatchSize)
|
||||
}
|
||||
if tc.Concurrency != m.Concurrency {
|
||||
t.Fatalf("--concurrency expected: %v got: %v", tc.Concurrency, m.Concurrency)
|
||||
}
|
||||
if tc.IDField != m.IDField {
|
||||
t.Fatalf("--id-field expected: %v got: %v", tc.IDField, m.IDField)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PilosaHosts, m.PilosaHosts) {
|
||||
t.Fatalf("--pilosa-hosts expected: %v got: %v", tc.PilosaHosts, m.PilosaHosts)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PilosaGRPCHosts, m.PilosaGRPCHosts) {
|
||||
t.Fatalf("--pilosa-grpc-hosts expected: %v got: %v", tc.PilosaGRPCHosts, m.PilosaGRPCHosts)
|
||||
}
|
||||
if tc.Stats != m.Stats {
|
||||
t.Fatalf("--stats expected: %v got: %v", tc.Stats, m.Stats)
|
||||
}
|
||||
if tc.Verbose != m.Verbose {
|
||||
t.Fatalf("--verbose expected: %v got: %v", tc.Verbose, m.Verbose)
|
||||
}
|
||||
if tc.ExpSplitBatchMode != m.ExpSplitBatchMode {
|
||||
t.Fatalf("--exp-split-batch-mode expected: %v got: %v", tc.ExpSplitBatchMode, m.ExpSplitBatchMode)
|
||||
}
|
||||
if tc.WriteCSV != m.WriteCSV {
|
||||
t.Fatalf("--write-csv expected: %v got: %v", tc.WriteCSV, m.WriteCSV)
|
||||
}
|
||||
if tc.Index != m.Index {
|
||||
t.Fatalf("--index expected: %v got: %v", tc.Index, m.Index)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PrimaryKeyFields, m.PrimaryKeyFields) {
|
||||
t.Fatalf("--primary-key-fields expected: %v got: %v", tc.PrimaryKeyFields, m.PrimaryKeyFields)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.KafkaHosts, m.KafkaBootstrapServers) {
|
||||
t.Fatalf("--kafka-hosts expected: %v got: %v", tc.KafkaHosts, m.KafkaBootstrapServers)
|
||||
}
|
||||
if tc.RegistryURL != m.SchemaRegistryURL {
|
||||
t.Fatalf("--registry-url expected: %v got: %v", tc.RegistryURL, m.SchemaRegistryURL)
|
||||
}
|
||||
if tc.ConfluentCommand.KafkaDebug != m.ConfluentCommand.KafkaDebug {
|
||||
t.Fatalf("--kafka-debug expected: %v got: %v", tc.ConfluentCommand.KafkaDebug, m.ConfluentCommand.KafkaDebug)
|
||||
}
|
||||
if tc.Group != m.Group {
|
||||
t.Fatalf("--group expected: %v got: %v", tc.Group, m.Group)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.Topics, m.Topics) {
|
||||
t.Fatalf("--topics expected: %v got: %v", tc.Topics, m.Topics)
|
||||
}
|
||||
if tc.Timeout != m.Timeout {
|
||||
t.Fatalf("--timeout expected: %v got: %v", tc.Timeout, m.Timeout)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
32
idk/cmd/molecula-consumer-kinesis/main.go
Normal file
32
idk/cmd/molecula-consumer-kinesis/main.go
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/kinesis"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
m := kinesis.NewMain()
|
||||
if err := pflag.LoadEnv(m, "CONSUMER_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
m.Rename()
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
if err := m.Run(); err != nil {
|
||||
log := m.Log()
|
||||
if log == nil {
|
||||
// if we fail before a logger was instantiated
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
38
idk/cmd/molecula-consumer-sql/main.go
Normal file
38
idk/cmd/molecula-consumer-sql/main.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/sql"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
func main() {
|
||||
m := sql.NewMain()
|
||||
if err := pflag.LoadEnv(m, "IDK_", nil); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
m.Rename()
|
||||
if m.DryRun {
|
||||
log.Printf("%+v\n", m)
|
||||
return
|
||||
}
|
||||
|
||||
if m.Concurrency != 1 {
|
||||
m.Log().Infof("Concurrency is not supported for sql ingest. '--concurrency' flag will be ignored.")
|
||||
m.Concurrency = 1
|
||||
}
|
||||
|
||||
if err := m.Run(); err != nil {
|
||||
log := m.Log()
|
||||
if log == nil {
|
||||
// if we fail before a logger was instantiated
|
||||
logger.NewStandardLogger(os.Stderr).Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
log.Errorf("Error running command: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
151
idk/cmd/molecula-consumer-sql/main_test.go
Normal file
151
idk/cmd/molecula-consumer-sql/main_test.go
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/jaffee/commandeer"
|
||||
"github.com/jaffee/commandeer/pflag"
|
||||
"github.com/molecula/featurebase/v3/idk/sql"
|
||||
pflag13 "github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func TestConsumerSQLArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
|
||||
AssumeEmptyPilosa bool
|
||||
AutoGenerate bool
|
||||
BatchSize int
|
||||
Concurrency int
|
||||
IDField string
|
||||
PilosaHosts []string
|
||||
PilosaGRPCHosts []string
|
||||
Stats string
|
||||
Verbose bool
|
||||
ExpSplitBatchMode bool
|
||||
WriteCSV string
|
||||
Driver string
|
||||
ConnectionString string
|
||||
RowExpr string
|
||||
StringArraySeparator string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{
|
||||
"", // os.Args[0] can be ignored
|
||||
},
|
||||
BatchSize: 1,
|
||||
Concurrency: 1,
|
||||
PilosaHosts: []string{"localhost:10101"},
|
||||
PilosaGRPCHosts: []string{"localhost:20101"},
|
||||
Stats: "localhost:9093",
|
||||
Driver: "postgres",
|
||||
ConnectionString: "postgres://user:password@localhost:5432/defaultindex?sslmode=disable",
|
||||
StringArraySeparator: ",",
|
||||
},
|
||||
{
|
||||
name: "long",
|
||||
args: []string{
|
||||
"molecula-consumer-sql",
|
||||
"--assume-empty-pilosa", "true",
|
||||
"--auto-generate", "true",
|
||||
"--batch-size", "12345",
|
||||
"--concurrency", "1",
|
||||
"--exp-split-batch-mode", "true",
|
||||
"--id-field", "id",
|
||||
"--index", "index_name",
|
||||
"--log-path", "/tmp/file.log",
|
||||
"--pack-bools", "true",
|
||||
"--pilosa-grpc-hosts", "grpc:1,grpc:2,grpc:3",
|
||||
"--pilosa-hosts", "pilosa:1,pilosa:2,pilosa:3",
|
||||
"--primary-key-fields", "primary_key",
|
||||
"--stats", "localhost:9093",
|
||||
"--tls.ca-certificate", "/tmp/file.ca",
|
||||
"--tls.certificate", "/tmp/file.certificate",
|
||||
"--tls.enable-client-verification", "true",
|
||||
"--tls.key", "/tmp/file.key",
|
||||
"--tls.skip-verify", "false",
|
||||
"--verbose", "true",
|
||||
"--write-csv", "/tmp/file.csv",
|
||||
"--driver", "mysql",
|
||||
"--connection-string", "mysql://user:password@localhost:3306/index_name",
|
||||
"--string-array-separator", ";",
|
||||
"--row-expr", "select 1;",
|
||||
},
|
||||
AssumeEmptyPilosa: true,
|
||||
AutoGenerate: true,
|
||||
BatchSize: 12345,
|
||||
Concurrency: 1,
|
||||
IDField: "id",
|
||||
PilosaHosts: []string{"pilosa:1", "pilosa:2", "pilosa:3"},
|
||||
PilosaGRPCHosts: []string{"grpc:1", "grpc:2", "grpc:3"},
|
||||
Stats: "localhost:9093",
|
||||
Verbose: true,
|
||||
ExpSplitBatchMode: true,
|
||||
WriteCSV: "/tmp/file.csv",
|
||||
Driver: "mysql",
|
||||
ConnectionString: "mysql://user:password@localhost:3306/index_name",
|
||||
StringArraySeparator: ";",
|
||||
RowExpr: "select 1;",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fs := &pflag.FlagSet{FlagSet: pflag13.NewFlagSet(tc.args[0], pflag13.ExitOnError)}
|
||||
m := sql.NewMain()
|
||||
|
||||
if err := commandeer.LoadArgsEnv(fs, m, tc.args[1:], "IDK_", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if tc.AssumeEmptyPilosa != m.AssumeEmptyPilosa {
|
||||
t.Fatalf("--assume-empty-pilosa expected: %v got: %v", tc.AssumeEmptyPilosa, m.AssumeEmptyPilosa)
|
||||
}
|
||||
if tc.AutoGenerate != m.AutoGenerate {
|
||||
t.Fatalf("--auto-generate expected: %v got: %v", tc.AutoGenerate, m.AutoGenerate)
|
||||
}
|
||||
if tc.BatchSize != m.BatchSize {
|
||||
t.Fatalf("--batch-size expected: %v got: %v", tc.BatchSize, m.BatchSize)
|
||||
}
|
||||
if tc.Concurrency != m.Concurrency {
|
||||
t.Fatalf("--concurrency expected: %v got: %v", tc.Concurrency, m.Concurrency)
|
||||
}
|
||||
if tc.IDField != m.IDField {
|
||||
t.Fatalf("--id-field expected: %v got: %v", tc.IDField, m.IDField)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PilosaHosts, m.PilosaHosts) {
|
||||
t.Fatalf("--pilosa-hosts expected: %v got: %v", tc.PilosaHosts, m.PilosaHosts)
|
||||
}
|
||||
if !reflect.DeepEqual(tc.PilosaGRPCHosts, m.PilosaGRPCHosts) {
|
||||
t.Fatalf("--pilosa-grpc-hosts expected: %v got: %v", tc.PilosaGRPCHosts, m.PilosaGRPCHosts)
|
||||
}
|
||||
if tc.Stats != m.Stats {
|
||||
t.Fatalf("--stats expected: %v got: %v", tc.Stats, m.Stats)
|
||||
}
|
||||
if tc.Verbose != m.Verbose {
|
||||
t.Fatalf("--verbose expected: %v got: %v", tc.Verbose, m.Verbose)
|
||||
}
|
||||
if tc.ExpSplitBatchMode != m.ExpSplitBatchMode {
|
||||
t.Fatalf("--exp-split-batch-mode expected: %v got: %v", tc.ExpSplitBatchMode, m.ExpSplitBatchMode)
|
||||
}
|
||||
if tc.WriteCSV != m.WriteCSV {
|
||||
t.Fatalf("--write-csv expected: %v got: %v", tc.WriteCSV, m.WriteCSV)
|
||||
}
|
||||
if tc.Driver != m.Driver {
|
||||
t.Fatalf("--driver expected: %v got: %v", tc.Driver, m.Driver)
|
||||
}
|
||||
if tc.ConnectionString != m.ConnectionString {
|
||||
t.Fatalf("--connection-string expected: %v got: %v", tc.ConnectionString, m.ConnectionString)
|
||||
}
|
||||
if tc.StringArraySeparator != m.StringArraySeparator {
|
||||
t.Fatalf("--string-array-separator expected: %v got: %v", tc.StringArraySeparator, m.StringArraySeparator)
|
||||
}
|
||||
if tc.RowExpr != m.RowExpr {
|
||||
t.Fatalf("--row-expr expected: %v got: %v", tc.RowExpr, m.RowExpr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
191
idk/common/configMap.go
Normal file
191
idk/common/configMap.go
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
package common
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
confluent "github.com/confluentinc/confluent-kafka-go/kafka"
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// LaunchKafkaEventConfirmer consumes the events channel for the messages Produced by the producer
|
||||
|
||||
func LaunchKafkaEventConfirmer(producer *confluent.Producer, finished *int32, iter *int64) chan struct{} {
|
||||
doneChan := make(chan struct{})
|
||||
go func() {
|
||||
defer func() {
|
||||
close(doneChan)
|
||||
}()
|
||||
cnt := int64(0)
|
||||
for e := range producer.Events() {
|
||||
switch ev := e.(type) {
|
||||
case *confluent.Message:
|
||||
cnt += 1
|
||||
msg := ev
|
||||
if msg.TopicPartition.Error != nil {
|
||||
fmt.Printf("Delivery failed: %v\n", msg.TopicPartition.Error)
|
||||
}
|
||||
default:
|
||||
fmt.Printf("Ignored event: %s\n", ev)
|
||||
}
|
||||
check := atomic.LoadInt32(finished)
|
||||
if check > 0 {
|
||||
sent := atomic.LoadInt64(iter)
|
||||
if cnt >= sent {
|
||||
doneChan <- struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return doneChan
|
||||
}
|
||||
func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) {
|
||||
var err error
|
||||
configMap := &confluent.ConfigMap{}
|
||||
if m.KafkaConfiguration != "" {
|
||||
file, er := ioutil.ReadFile(m.KafkaConfiguration)
|
||||
if er != nil {
|
||||
return nil, er
|
||||
}
|
||||
er = json.Unmarshal([]byte(file), configMap)
|
||||
if er != nil {
|
||||
return nil, er
|
||||
}
|
||||
}
|
||||
err = configMap.SetKey("enable.auto.commit", false)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "disabling auto commit")
|
||||
}
|
||||
if m.KafkaSocketTimeoutMs > 0 {
|
||||
err = configMap.SetKey("socket.timeout.ms", m.KafkaSocketTimeoutMs)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "setting socket timeout")
|
||||
}
|
||||
} else {
|
||||
err = configMap.SetKey("socket.timeout.ms", 60000)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "setting socket timeout")
|
||||
}
|
||||
}
|
||||
if len(m.KafkaBootstrapServers) > 0 {
|
||||
err = configMap.SetKey("bootstrap.servers", strings.Join(m.KafkaBootstrapServers, ","))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "setting bootstrap")
|
||||
}
|
||||
|
||||
} else {
|
||||
err = configMap.SetKey("bootstrap.servers", "localhost:9092")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "setting bootstrap")
|
||||
}
|
||||
}
|
||||
|
||||
//SSL
|
||||
if m.KafkaSslCaLocation != "" {
|
||||
err = configMap.SetKey("ssl.ca.location", m.KafkaSslCaLocation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.KafkaSslKeyLocation != "" {
|
||||
err = configMap.SetKey("ssl.key.location", m.KafkaSslKeyLocation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.KafkaSslKeyPassword != "" {
|
||||
err = configMap.SetKey("ssl.key.password", m.KafkaSslKeyPassword)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.KafkaSslCertificateLocation != "" {
|
||||
err = configMap.SetKey("ssl.certificate.location", m.KafkaSslCertificateLocation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
err = configMap.SetKey("enable.ssl.certificate.verification", m.KafkaEnableSslCertificateVerification)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if m.KafkaSslEndpointIdentificationAlgorithm != "" {
|
||||
err = configMap.SetKey("ssl.endpoint.identification.algorithm", m.KafkaSslEndpointIdentificationAlgorithm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
//SSL
|
||||
if m.KafkaSslCaLocation != "" {
|
||||
err = configMap.SetKey("ssl.ca.location", m.KafkaSslCaLocation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.KafkaSslKeyLocation != "" {
|
||||
err = configMap.SetKey("ssl.key.location", m.KafkaSslKeyLocation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.KafkaSslCertificateLocation != "" {
|
||||
err = configMap.SetKey("ssl.certificate.location", m.KafkaSslCertificateLocation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
err = configMap.SetKey("enable.ssl.certificate.verification", m.KafkaEnableSslCertificateVerification)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if m.KafkaSslEndpointIdentificationAlgorithm != "" {
|
||||
err = configMap.SetKey("ssl.endpoint.identification.algorithm", m.KafkaSslEndpointIdentificationAlgorithm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// SASL,
|
||||
if m.KafkaSaslUsername != "" && m.KafkaSaslPassword != "" {
|
||||
err = configMap.SetKey("sasl.username", m.KafkaSaslUsername)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = configMap.SetKey("sasl.password", m.KafkaSaslPassword)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.KafkaSaslMechanism != "" {
|
||||
err = configMap.SetKey("sasl.mechanism", m.KafkaSaslMechanism)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.KafkaSecurityProtocol != "" {
|
||||
err = configMap.SetKey("security.protocol", m.KafkaSecurityProtocol)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.KafkaClientId != "" {
|
||||
err = configMap.SetKey("client.id", m.KafkaClientId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.KafkaDebug != "" {
|
||||
err = configMap.SetKey("debug", m.KafkaDebug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return configMap, nil
|
||||
}
|
||||
138
idk/csv/cmd.go
Normal file
138
idk/csv/cmd.go
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
package csv
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type Main struct {
|
||||
idk.Main `flag:"!embed"`
|
||||
Files []string `help:"List of files, URLs, or directories to ingest."`
|
||||
Header []string `help:"Optional header. If not passed, first line of each file is used."`
|
||||
IgnoreHeader bool `short:"g" help:"Ignore header in file and use configured header. You *must* configure a header."`
|
||||
JustDoIt bool `short:"j" help:"Any header field not in the appropriate format, just downcase, use it as the name and process the value as a String/set field"`
|
||||
|
||||
files chan string
|
||||
visited map[string]struct{}
|
||||
}
|
||||
|
||||
func NewMain() *Main {
|
||||
m := &Main{
|
||||
Main: *idk.NewMain(),
|
||||
files: make(chan string),
|
||||
visited: make(map[string]struct{}),
|
||||
}
|
||||
m.Main.Namespace = "ingester_csv"
|
||||
|
||||
once := &sync.Once{}
|
||||
m.NewSource = func() (idk.Source, error) {
|
||||
if len(m.Files) == 0 {
|
||||
return nil, errors.New("must provide at least one file or directory with --files")
|
||||
}
|
||||
once.Do(func() { go m.streamFileNames() })
|
||||
|
||||
source := NewSource()
|
||||
source.Files = m.files
|
||||
source.Header = m.Header
|
||||
source.Log = m.Main.Log()
|
||||
source.once = &sync.Once{}
|
||||
source.JustDoIt = m.JustDoIt
|
||||
source.IgnoreHeader = m.IgnoreHeader
|
||||
return source, nil
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// ValidateHeaders is used during --dry-run to check that all headers provided at CLI or in
|
||||
// CSV files can be parsed properly.
|
||||
// Also returns the field structs as it would be parsed in a normal run. In the case of
|
||||
// multiple CSV files, only returns the fields for the last file.
|
||||
func (m *Main) ValidateHeaders() ([]idk.Field, error) {
|
||||
var err error
|
||||
fields := make([]idk.Field, 0)
|
||||
if len(m.Header) > 0 {
|
||||
fields, err = m.validateHeader(m.Header)
|
||||
if err != nil {
|
||||
return fields, errors.Wrap(err, "validating header")
|
||||
}
|
||||
} else {
|
||||
for _, f := range m.Files {
|
||||
f, err := openFileOrURL(f)
|
||||
if err != nil {
|
||||
return fields, errors.Wrapf(err, "opening file (%s)", f)
|
||||
}
|
||||
defer f.Close()
|
||||
reader := csv.NewReader(f)
|
||||
header, err := reader.Read()
|
||||
if err != nil {
|
||||
return fields, errors.Wrapf(err, "reading CSV file (%s)", f)
|
||||
}
|
||||
fields, err = m.validateHeader(header)
|
||||
if err != nil {
|
||||
return fields, errors.Wrap(err, "validating header")
|
||||
}
|
||||
}
|
||||
}
|
||||
return fields, nil
|
||||
}
|
||||
|
||||
func (m *Main) validateHeader(h []string) ([]idk.Field, error) {
|
||||
fields := make([]idk.Field, 0)
|
||||
for _, val := range h {
|
||||
field, err := idk.HeaderToField(val, m.Log())
|
||||
if err != nil {
|
||||
return fields, errors.Wrapf(err, "invalid header (%v)", val)
|
||||
}
|
||||
fields = append(fields, field)
|
||||
}
|
||||
return fields, nil
|
||||
}
|
||||
|
||||
func (m *Main) streamFileNames() {
|
||||
defer close(m.files)
|
||||
|
||||
for _, filename := range m.Files {
|
||||
if err := filepath.Walk(filename, m.traverse); err != nil {
|
||||
m.Log().Printf("filepath.Walk(%s) %v", filename, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Main) traverse(path string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
m.Log().Printf("prevent panic by handling failure accessing a path %q: %v\n", path, err)
|
||||
return err
|
||||
}
|
||||
|
||||
switch mode := fi.Mode(); {
|
||||
case mode.IsRegular():
|
||||
if _, ok := m.visited[path]; !ok {
|
||||
m.visited[path] = struct{}{}
|
||||
m.files <- path
|
||||
}
|
||||
|
||||
case mode.IsDir():
|
||||
if _, ok := m.visited[path]; !ok {
|
||||
m.visited[path] = struct{}{}
|
||||
}
|
||||
|
||||
case mode&os.ModeSymlink != 0:
|
||||
lnk, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, ok := m.visited[lnk]; !ok {
|
||||
if err := filepath.Walk(lnk, m.traverse); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
654
idk/csv/csv_test.go
Normal file
654
idk/csv/csv_test.go
Normal file
|
|
@ -0,0 +1,654 @@
|
|||
package csv
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk/idktest"
|
||||
)
|
||||
|
||||
func configureTestFlags(main *Main) {
|
||||
if pilosaHost, ok := os.LookupEnv("IDK_TEST_PILOSA_HOST"); ok {
|
||||
main.PilosaHosts = []string{pilosaHost}
|
||||
} else {
|
||||
main.PilosaHosts = []string{"pilosa:10101"}
|
||||
}
|
||||
if grpcHost, ok := os.LookupEnv("IDK_TEST_GRPC_HOST"); ok {
|
||||
main.PilosaGRPCHosts = []string{grpcHost}
|
||||
} else {
|
||||
main.PilosaGRPCHosts = []string{"pilosa:20101"}
|
||||
}
|
||||
main.Stats = ""
|
||||
}
|
||||
|
||||
func TestCSVCommand(t *testing.T) {
|
||||
file := `
|
||||
id__ID,s__String_F_YMDH,__RecordTime_2006-01-02T15
|
||||
0,a,2019-01-09T04
|
||||
1,a,2019-01-09T05
|
||||
2,a,2019-02-09T04
|
||||
3,b,2019-01-09T04
|
||||
4,a,2018-01-09T04
|
||||
5,a,2019-01-10T04
|
||||
`[1:]
|
||||
name := writeTempFile(t, file)
|
||||
|
||||
m := NewMain()
|
||||
m.AutoGenerate = true
|
||||
configureTestFlags(m)
|
||||
m.Files = []string{name}
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
m.Index = fmt.Sprintf("csvtest%d", rand.Intn(100000))
|
||||
|
||||
defer func() {
|
||||
if err := m.PilosaClient().DeleteIndexByName(m.Index); err != nil {
|
||||
t.Logf("deleting test index: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err := m.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("running: %v", err)
|
||||
}
|
||||
|
||||
client := m.PilosaClient()
|
||||
|
||||
schema, err := client.Schema()
|
||||
if err != nil {
|
||||
t.Fatalf("getting schema: %v", err)
|
||||
}
|
||||
index := schema.Index(m.Index)
|
||||
|
||||
s := index.Field("s")
|
||||
|
||||
resp, err := client.Query(s.Range("a", tim(t, "2019-01-07T03"), tim(t, "2019-01-10T05")))
|
||||
if err != nil {
|
||||
t.Fatalf("querying: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(resp.Results()[0].Row().Columns, []uint64{0, 1, 5}) {
|
||||
t.Errorf("got/exp\n%v\n%v", resp.Results()[0].Row().Columns, []uint64{0, 1, 5})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVCommandCustomHeader(t *testing.T) {
|
||||
file := `
|
||||
ABCD,2019-01-02,70%
|
||||
ABCD,2019-01-03,20%
|
||||
ABCD,2019-01-04,30%
|
||||
BEDF,2019-01-02,70%
|
||||
BEDF,2019-01-05,90%
|
||||
BEDF,2019-01-08,10%
|
||||
BEDF,2019-01-08,20%
|
||||
ABCD,2019-01-30,40%
|
||||
`[1:]
|
||||
name := writeTempFile(t, file)
|
||||
|
||||
m := NewMain()
|
||||
m.AutoGenerate = true
|
||||
m.Header = []string{"asset_tag__String", "fan_time__RecordTime_2006-01-02"}
|
||||
configureTestFlags(m)
|
||||
m.Files = []string{name}
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
m.Index = fmt.Sprintf("csvtest%d", rand.Intn(100000))
|
||||
|
||||
defer func() {
|
||||
if err := m.PilosaClient().DeleteIndexByName(m.Index); err != nil {
|
||||
t.Logf("deleting test index: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err := m.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("running: %v", err)
|
||||
}
|
||||
|
||||
client := m.PilosaClient()
|
||||
|
||||
schema, err := client.Schema()
|
||||
if err != nil {
|
||||
t.Fatalf("getting schema: %v", err)
|
||||
}
|
||||
index := schema.Index(m.Index)
|
||||
|
||||
s := index.Field("asset_tag")
|
||||
|
||||
resp, err := client.Query(s.Row("ABCD"))
|
||||
if err != nil {
|
||||
t.Fatalf("querying: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(resp.Results()[0].Row().Columns, []uint64{0, 1, 2, 7}) {
|
||||
t.Errorf("got/exp\n%v\n%v", resp.Results()[0].Row().Columns, []uint64{0, 1, 2, 7})
|
||||
}
|
||||
}
|
||||
|
||||
type recordTimeTestCase struct {
|
||||
name string
|
||||
layout string
|
||||
unit string
|
||||
scale int // how many unit it takes to move an hour
|
||||
}
|
||||
|
||||
func TestCSVRecordTime(t *testing.T) {
|
||||
testCases := []recordTimeTestCase{
|
||||
{name: "plusHours", layout: "", unit: "h", scale: 1},
|
||||
{name: "plusMinutes", layout: "", unit: "m", scale: 60},
|
||||
{name: "plusSeconds", layout: "", unit: "s", scale: 60 * 60},
|
||||
}
|
||||
// We set our epoch at 4AM, 2019-01-09, because that's the epoch used in
|
||||
// the previous example.
|
||||
arbitraryEpoch, err := time.Parse("2006-01-02T15", "2019-01-09T04")
|
||||
if err != nil {
|
||||
t.Fatalf("parsing time: %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
lines := []string{}
|
||||
layout := tc.layout
|
||||
if layout == "" {
|
||||
layout = time.RFC3339
|
||||
}
|
||||
header := fmt.Sprintf(`id__ID,s__String_F_YMDH,__RecordTime_%s_%s_%s`,
|
||||
layout, arbitraryEpoch.Format(layout), tc.unit)
|
||||
lines = append(lines, header)
|
||||
// We expect -1..1 to show up in our query, and -2 and +2 not to.
|
||||
for i, offset := range []int{-2, -1, 0, 1, 2} {
|
||||
record := fmt.Sprintf("%d,a,%d", i, offset*tc.scale)
|
||||
lines = append(lines, record)
|
||||
}
|
||||
lines = append(lines, "")
|
||||
file := strings.Join(lines, "\n")
|
||||
|
||||
name := writeTempFile(t, file)
|
||||
|
||||
m := NewMain()
|
||||
m.AutoGenerate = true
|
||||
configureTestFlags(m)
|
||||
m.Files = []string{name}
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
m.Index = fmt.Sprintf("csvtest%d", rand.Intn(100000))
|
||||
|
||||
defer func() {
|
||||
if err := m.PilosaClient().DeleteIndexByName(m.Index); err != nil {
|
||||
t.Logf("deleting test index: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err := m.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("running: %v", err)
|
||||
}
|
||||
|
||||
client := m.PilosaClient()
|
||||
|
||||
schema, err := client.Schema()
|
||||
if err != nil {
|
||||
t.Fatalf("getting schema: %v", err)
|
||||
}
|
||||
index := schema.Index(m.Index)
|
||||
|
||||
s := index.Field("s")
|
||||
|
||||
// We specify a range of one minute after the hour because otherwise
|
||||
// we hit the end of the range and don't count the +1 case. I am
|
||||
// not convinced that this is correct.
|
||||
resp, err := client.Query(s.Range("a", tim(t, "2019-01-09T03"), tim(t, "2019-01-09T05").Add(1*time.Minute)))
|
||||
if err != nil {
|
||||
t.Fatalf("querying: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(resp.Results()[0].Row().Columns, []uint64{1, 2, 3}) {
|
||||
t.Errorf("record time (%s): got/exp\n%v\n%v", tc.name, resp.Results()[0].Row().Columns, []uint64{1, 2, 3})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func tim(t *testing.T, tstr string) time.Time {
|
||||
ti, err := time.Parse("2006-01-02T15", tstr)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing time: %v", err)
|
||||
}
|
||||
return ti
|
||||
}
|
||||
|
||||
func writeTempFile(t *testing.T, data string) string {
|
||||
f, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
t.Fatalf("getting temp file: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
_, err = io.WriteString(f, data)
|
||||
if err != nil {
|
||||
t.Fatalf("writing string: %v", err)
|
||||
}
|
||||
|
||||
return f.Name()
|
||||
}
|
||||
|
||||
func TestStreamFileNames(t *testing.T) {
|
||||
tmp, err := testDirTree("d1/d2/d3/d4/d5")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
|
||||
mtx := &sync.Mutex{}
|
||||
visited := make(map[string]int)
|
||||
|
||||
m := NewMain()
|
||||
configureTestFlags(m)
|
||||
// duplicate all files from the same directory
|
||||
m.Files = []string{tmp, tmp, tmp, tmp, tmp, tmp}
|
||||
_, err = m.NewSource()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
visitor := func(files chan string) {
|
||||
defer wg.Done()
|
||||
|
||||
for f := range files {
|
||||
// count all visited files
|
||||
mtx.Lock()
|
||||
visited[f]++
|
||||
mtx.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
wg.Add(3)
|
||||
go visitor(m.files)
|
||||
go visitor(m.files)
|
||||
go visitor(m.files)
|
||||
wg.Wait()
|
||||
|
||||
for p, n := range visited {
|
||||
fi, err := os.Lstat(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if fi.Mode().IsRegular() && n != 1 {
|
||||
t.Fatalf("file %s was visited %d times", p, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testDirTree(tree string) (string, error) {
|
||||
tmp, err := ioutil.TempDir("", "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
err = os.MkdirAll(filepath.Join(tmp, tree), 0755)
|
||||
if err != nil {
|
||||
os.RemoveAll(tmp)
|
||||
return "", err
|
||||
}
|
||||
|
||||
return tmp, filepath.Walk(tmp, func(path string, _ os.FileInfo, _ error) error {
|
||||
f, err := os.Create(filepath.Join(path, "csv"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
})
|
||||
}
|
||||
|
||||
// Tests various out of range values are ingested as nil when CLI flags are set to true.
|
||||
func TestVariousOORValues(t *testing.T) {
|
||||
file := `
|
||||
id__ID,s__String_F_YMDH,ts__Timestamp_s_2006-01-02 15:04:05.999,price__Decimal_2,age__Int_1_120
|
||||
0,a,1832-01-03 08:00:00.000,0.0,1
|
||||
1,b,1700-01-03 08:00:00.000,5.44,35
|
||||
2,b,2019-50-03 08:00:00.000,5.44,120
|
||||
3,b,2019-01-50 08:00:00.000,5.44,120
|
||||
4,b,2019-01-03 50:00:00.000,5.44,120
|
||||
5,a,2019-04-03 00:90:00.000,5.44,129
|
||||
6,a,2019-04-03 00:00:90.000,5.44,120
|
||||
5,a,2019-04-03 00:00:00.000,123.123,1
|
||||
6,a,2019-04-03 00:00:00.000,-1,1
|
||||
7,a,2019-04-03 00:00:00.000,994492233720368547758.0892233720368547758,100
|
||||
5,a,2019-04-03 00:00:00.000,2.34,121
|
||||
6,a,2019-04-03 00:00:00.000,3.44,0
|
||||
7,a,1500-04-03 00:00:00.000,994492233720368547758.0892233720368547758,2342342
|
||||
8,a,2019-04-03 00:00:00.000,3.44,100
|
||||
`[1:]
|
||||
name := writeTempFile(t, file)
|
||||
|
||||
checker := make(map[string]interface{})
|
||||
checker["ts"] = []interface{}{nil, nil, nil, nil, nil, nil, nil, "2019-04-03T00:00:00Z", "2019-04-03T00:00:00Z", "2019-04-03T00:00:00Z", "2019-04-03T00:00:00Z", "2019-04-03T00:00:00Z", nil, "2019-04-03T00:00:00Z"}
|
||||
checker["age"] = []interface{}{1, 35, 120, 120, 120, nil, 120, 1, 1, 100, nil, nil, nil, 100}
|
||||
checker["price"] = []interface{}{0.0, 5.44, 5.44, 5.44, 5.44, 5.44, 5.44, 123.12, -1.0, nil, 2.34, 3.44, nil, 3.44}
|
||||
|
||||
m := NewMain()
|
||||
m.AutoGenerate = true
|
||||
m.BatchSize = 1
|
||||
configureTestFlags(m)
|
||||
m.Files = []string{name}
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
m.Index = fmt.Sprintf("csvtest%d", rand.Intn(100000))
|
||||
m.AllowIntOutOfRange = true
|
||||
m.AllowDecimalOutOfRange = true
|
||||
m.AllowTimestampOutOfRange = true
|
||||
|
||||
defer func() {
|
||||
if err := m.PilosaClient().DeleteIndexByName(m.Index); err != nil {
|
||||
t.Logf("deleting test index: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err := m.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("running: %v", err)
|
||||
}
|
||||
|
||||
client := m.PilosaClient()
|
||||
|
||||
schema, err := client.Schema()
|
||||
if err != nil {
|
||||
t.Fatalf("getting schema: %v", err)
|
||||
}
|
||||
index := schema.Index(m.Index)
|
||||
|
||||
resp, err := client.Query(index.Count(index.All()))
|
||||
if err != nil {
|
||||
t.Fatalf("querying: %v", err)
|
||||
}
|
||||
if cnt := resp.Result().Count(); cnt != 14 {
|
||||
t.Fatalf("expected 14, got %+v", cnt)
|
||||
}
|
||||
|
||||
for _, field := range []string{"ts", "age", "price"} {
|
||||
pql := fmt.Sprintf("Extract(All(), Rows(%s))", field)
|
||||
|
||||
eResp, err := idktest.DoExtractQuery(pql, m.Index)
|
||||
if err != nil {
|
||||
t.Fatal("doing extract: ", err)
|
||||
}
|
||||
if eResp.Results[0].Columns == nil {
|
||||
t.Fatal("no results: ", err)
|
||||
}
|
||||
|
||||
for i, item := range eResp.Results[0].Columns {
|
||||
check := checker[field].([]interface{})
|
||||
|
||||
switch exp := check[i].(type) {
|
||||
case nil:
|
||||
if item.Rows[0] != nil {
|
||||
t.Errorf("expected nil, got %+v", item.Rows[0])
|
||||
}
|
||||
case string:
|
||||
if item.Rows[0] != exp {
|
||||
t.Errorf("expected %s, got %+v", exp, item.Rows[0])
|
||||
}
|
||||
case int:
|
||||
vAsInt := int(item.Rows[0].(float64))
|
||||
if vAsInt != exp {
|
||||
t.Errorf("expected %d, got %+v", exp, item.Rows[0])
|
||||
} else {
|
||||
expAsFloat := float64(exp)
|
||||
if item.Rows[0] != expAsFloat {
|
||||
t.Errorf("expected %f, got %+v", expAsFloat, item.Rows[0])
|
||||
}
|
||||
}
|
||||
case float64:
|
||||
if item.Rows[0] != exp {
|
||||
t.Errorf("expected %f, got %+v", exp, item.Rows[0])
|
||||
}
|
||||
default:
|
||||
t.Errorf("unknown type: %T", exp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Test that out of range int values are ingested as nil when AllowIntOutOfRange is true.
|
||||
func TestIntOpts(t *testing.T) {
|
||||
file := `
|
||||
id__ID,negneg__Int_-10_-5,negpos__Int_-10_10,pospos__Int_5_10,negzero__Int_-10_0,zeropos__Int_0_10,zerozero__Int_0_0
|
||||
1,-20,-20,-20,-20,-20,-20
|
||||
2,-10,-10,-10,-10,-10,-10
|
||||
3,-5,-5,-5,-5,-5,-5
|
||||
4,0,0,0,0,0,0
|
||||
5,5,5,5,5,5,5
|
||||
6,10,10,10,10,10,10
|
||||
7,20,20,20,20,20,20
|
||||
`[1:]
|
||||
|
||||
checker := make(map[string]interface{})
|
||||
checker["negneg"] = []interface{}{nil, -10, -5, nil, nil, nil, nil}
|
||||
checker["negzero"] = []interface{}{nil, -10, -5, 0, nil, nil, nil}
|
||||
checker["negpos"] = []interface{}{nil, -10, -5, 0, 5, 10, nil}
|
||||
checker["zeropos"] = []interface{}{nil, nil, nil, 0, 5, 10, nil}
|
||||
checker["zerozero"] = []interface{}{nil, nil, nil, 0, nil, nil, nil}
|
||||
checker["pospos"] = []interface{}{nil, nil, nil, nil, 5, 10, nil}
|
||||
|
||||
batchSizes := []int{1, 3, 4, 10}
|
||||
for _, bsize := range batchSizes {
|
||||
t.Run(fmt.Sprintf("batchsize=%d", bsize), func(t *testing.T) {
|
||||
m := newMainOORFactory(t, file, true, false, false)
|
||||
m.BatchSize = bsize
|
||||
|
||||
defer func() {
|
||||
if err := m.PilosaClient().DeleteIndexByName(m.Index); err != nil {
|
||||
t.Logf("deleting test index: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err := m.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("running: %v", err)
|
||||
}
|
||||
|
||||
for _, field := range []string{"negneg", "negzero", "negpos", "zeropos", "pospos"} {
|
||||
pql := fmt.Sprintf("Extract(All(), Rows(%s))", field)
|
||||
|
||||
eResp, err := idktest.DoExtractQuery(pql, m.Index)
|
||||
if err != nil {
|
||||
t.Fatal("doing extract: ", err)
|
||||
}
|
||||
if eResp.Results[0].Columns == nil {
|
||||
t.Fatal("no results: ", err)
|
||||
}
|
||||
|
||||
for i, item := range eResp.Results[0].Columns {
|
||||
check := checker[field].([]interface{})
|
||||
switch exp := check[i].(type) {
|
||||
case nil:
|
||||
if item.Rows[0] != nil {
|
||||
t.Errorf("expected nil, got %+v", item.Rows[0])
|
||||
}
|
||||
case string:
|
||||
if item.Rows[0] != exp {
|
||||
t.Errorf("expected %s, got %+v", exp, item.Rows[0])
|
||||
}
|
||||
case int:
|
||||
vAsInt := int(item.Rows[0].(float64))
|
||||
if vAsInt != exp {
|
||||
t.Errorf("expected %d, got %+v", exp, item.Rows[0])
|
||||
} else {
|
||||
expAsFloat := float64(exp)
|
||||
if item.Rows[0] != expAsFloat {
|
||||
t.Errorf("expected %f, got %+v", expAsFloat, item.Rows[0])
|
||||
}
|
||||
}
|
||||
case float64:
|
||||
if item.Rows[0] != exp {
|
||||
t.Errorf("expected %f, got %+v", exp, item.Rows[0])
|
||||
}
|
||||
default:
|
||||
t.Errorf("unknown type: %T", exp)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Test that out of range timestamp values are ingested as nil when AllowTimestampOutOfRange is true.
|
||||
func TestTimestampOOR(t *testing.T) {
|
||||
file := `
|
||||
id__ID,ts1__Timestamp_s_2006-01-02 15:04:05.999,ts2__Timestamp_s_2006-01-02T15:04:05Z07:00_2261-12-31T15:04:05Z_h,ts3__Timestamp_s_2006-01-02T15:04:05Z07:00_1679-12-31T15:04:05Z_h,ts4__Timestamp_s_2006-01-02 15:04:05.999
|
||||
0,1833-01-03 08:00:00.000,2431,-19960,2009-11-24 17:31:44.000
|
||||
1,1833-11-24 17:31:44.000,2433,-19955,1800-11-24 17:31:44.000
|
||||
2,1833-11-25 17:31:44.000,9999,0,2011-11-25 17:31:44.000
|
||||
3,2106-02-06 06:28:16.000,0,-99999,2008-02-06 06:28:16.000
|
||||
4,2106-02-07 06:28:16.000,9999,-99999,1800-02-07 06:28:16.000
|
||||
5,2106-02-08 06:28:16.000,99999999999999999999999,-9999999999999999999999999,2012-02-07 06:28:16.000
|
||||
`[1:]
|
||||
|
||||
checker := make(map[string]interface{})
|
||||
// should import nil if timestamp val is out of range
|
||||
checker["ts1"] = []interface{}{nil, "1833-11-24T17:31:44Z", "1833-11-25T17:31:44Z", "2106-02-06T06:28:16Z", "2106-02-07T06:28:16Z", nil}
|
||||
// should import nil if custom epoch + value overflows
|
||||
checker["ts2"] = []interface{}{"2262-04-11T22:04:05Z", nil, nil, "2261-12-31T15:04:05Z", nil, nil}
|
||||
checker["ts3"] = []interface{}{nil, "1677-09-21T04:04:05Z", "1679-12-31T15:04:05Z", nil, nil, nil}
|
||||
checker["ts4"] = []interface{}{"2009-11-24T17:31:44Z", nil, "2011-11-25T17:31:44Z", "2008-02-06T06:28:16Z", nil, "2012-02-07T06:28:16Z"}
|
||||
batchSizes := []int{3, 1, 4, 10}
|
||||
for _, bsize := range batchSizes {
|
||||
|
||||
t.Run(fmt.Sprintf("batchsize=%d", bsize), func(t *testing.T) {
|
||||
m := newMainOORFactory(t, file, false, false, true)
|
||||
m.BatchSize = bsize
|
||||
|
||||
defer func() {
|
||||
if err := m.PilosaClient().DeleteIndexByName(m.Index); err != nil {
|
||||
t.Logf("deleting test index: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err := m.Run()
|
||||
if err != nil {
|
||||
t.Fatalf("running: %v", err)
|
||||
}
|
||||
for _, field := range []string{"ts1", "ts2", "ts3", "ts4"} {
|
||||
pql := fmt.Sprintf("Extract(All(), Rows(%s))", field)
|
||||
|
||||
eResp, err := idktest.DoExtractQuery(pql, m.Index)
|
||||
if err != nil {
|
||||
t.Fatal("doing extract: ", err)
|
||||
}
|
||||
if eResp.Results[0].Columns == nil {
|
||||
t.Fatal("no results: ", err)
|
||||
}
|
||||
|
||||
for i, item := range eResp.Results[0].Columns {
|
||||
check := checker[field].([]interface{})
|
||||
|
||||
switch exp := check[i].(type) {
|
||||
case nil:
|
||||
if item.Rows[0] != nil {
|
||||
t.Errorf("expected nil, got %+v", item.Rows[0])
|
||||
}
|
||||
case string:
|
||||
if item.Rows[0] != exp {
|
||||
t.Errorf("expected %s, got %+v", exp, item.Rows[0])
|
||||
}
|
||||
default:
|
||||
t.Errorf("unknown type: %T", exp)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Tests various conditions that should halt ingest
|
||||
func TestFailureConditions(t *testing.T) {
|
||||
type testCase struct {
|
||||
name string
|
||||
csv string
|
||||
fail bool
|
||||
intOutOfRange bool
|
||||
decimalOutOfRange bool
|
||||
timestampOutOfRange bool
|
||||
}
|
||||
|
||||
testCases := []testCase{
|
||||
{name: "too small", csv: `id__ID,ts1__Timestamp_s_2006-01-02T15:04:05Z07:00_1600-12-31T15:04:05Z_h
|
||||
0,0
|
||||
`, fail: true, intOutOfRange: true, timestampOutOfRange: true, decimalOutOfRange: true},
|
||||
{name: "just right", csv: `id__ID,ts1__Timestamp_s_2006-01-02T15:04:05Z07:00_2200-12-31T15:04:05Z_h
|
||||
0,0
|
||||
`, fail: false, intOutOfRange: true, timestampOutOfRange: true, decimalOutOfRange: true},
|
||||
{name: "too big", csv: `id__ID,ts1__Timestamp_s_2006-01-02T15:04:05Z07:00_2262-12-31T15:04:05Z_h
|
||||
0,0
|
||||
`, fail: true, intOutOfRange: true, timestampOutOfRange: true, decimalOutOfRange: true},
|
||||
{name: "intOutOfRange not allowed", csv: `id__ID,pospos__Int_5_10
|
||||
0,4
|
||||
`, fail: true, intOutOfRange: false, timestampOutOfRange: true, decimalOutOfRange: true},
|
||||
{name: "intOutOfRange not allowed-2", csv: `id__ID,pospos__Int_5_10
|
||||
0,11
|
||||
`, fail: true, intOutOfRange: false, timestampOutOfRange: true, decimalOutOfRange: true},
|
||||
{name: "timestampOutOfRange not allowed", csv: `id__ID,ts1__Timestamp_s_2006-01-02 15:04:05.999
|
||||
0,1833-01-03 08:00:00.000
|
||||
`, fail: true, intOutOfRange: true, timestampOutOfRange: false, decimalOutOfRange: true},
|
||||
{name: "timestampOutOfRange not allowed-2", csv: `id__ID,ts2__Timestamp_s_2006-01-02T15:04:05Z07:00_2261-12-31T15:04:05Z_h
|
||||
0,2433
|
||||
`, fail: true, intOutOfRange: true, timestampOutOfRange: false, decimalOutOfRange: true},
|
||||
{name: "decimalOutOfRange not allowed", csv: `id__ID,price__Decimal_2
|
||||
0,994492233720368547758.0892233720368547758
|
||||
`, fail: true, intOutOfRange: true, timestampOutOfRange: true, decimalOutOfRange: false},
|
||||
{name: "intOverflow", csv: `id__ID,pospos__Int
|
||||
0,89273948723984729387492387492987
|
||||
`, fail: true, intOutOfRange: false, timestampOutOfRange: false, decimalOutOfRange: false},
|
||||
|
||||
{name: "int string overflow", csv: `id__ID,pospos__Int
|
||||
0,"89273948723984729387492387492987"
|
||||
`, fail: true, intOutOfRange: false, timestampOutOfRange: false, decimalOutOfRange: false}}
|
||||
|
||||
for _, test := range testCases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
m := newMainOORFactory(t, test.csv, test.intOutOfRange, test.decimalOutOfRange, test.timestampOutOfRange)
|
||||
m.BatchSize = 1
|
||||
|
||||
defer func() {
|
||||
if err := m.PilosaClient().DeleteIndexByName(m.Index); err != nil {
|
||||
t.Logf("deleting test index: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err := m.Run()
|
||||
if test.fail {
|
||||
if err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Fatalf("running: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func newMainOORFactory(t *testing.T, file string, allowIntOOR bool, allowDecOOR bool, allowTSOOR bool) *Main {
|
||||
name := writeTempFile(t, file)
|
||||
m := NewMain()
|
||||
m.AutoGenerate = true
|
||||
configureTestFlags(m)
|
||||
m.Files = []string{name}
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
m.Index = fmt.Sprintf("oortest%d", rand.Intn(100000))
|
||||
m.AllowIntOutOfRange = allowIntOOR
|
||||
m.AllowDecimalOutOfRange = allowDecOOR
|
||||
m.AllowTimestampOutOfRange = allowTSOOR
|
||||
return m
|
||||
}
|
||||
187
idk/csv/source.go
Normal file
187
idk/csv/source.go
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
package csv
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
Files chan string
|
||||
Header []string
|
||||
IgnoreHeader bool
|
||||
JustDoIt bool
|
||||
Log logger.Logger
|
||||
|
||||
schemaLock sync.Mutex
|
||||
schema []idk.Field
|
||||
|
||||
records chan Record
|
||||
once *sync.Once
|
||||
expectHeader bool
|
||||
}
|
||||
|
||||
func (s *Source) Record() (idk.Record, error) {
|
||||
s.once.Do(func() { go s.run() })
|
||||
|
||||
rec, ok := <-s.records
|
||||
if !ok {
|
||||
return nil, io.EOF
|
||||
}
|
||||
return rec, rec.err
|
||||
}
|
||||
|
||||
type Record struct {
|
||||
data []interface{}
|
||||
err error
|
||||
}
|
||||
|
||||
func (r Record) Data() []interface{} {
|
||||
return r.data
|
||||
}
|
||||
func (r Record) Commit(ctx context.Context) error { return nil } // TODO do
|
||||
|
||||
func (s *Source) Schema() []idk.Field {
|
||||
s.schemaLock.Lock()
|
||||
defer s.schemaLock.Unlock()
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *Source) run() {
|
||||
defer close(s.records)
|
||||
|
||||
s.expectHeader = len(s.Header) == 0
|
||||
if !s.expectHeader {
|
||||
var err error
|
||||
s.schemaLock.Lock()
|
||||
s.schema, err = s.processHeader(s.Header)
|
||||
s.schemaLock.Unlock()
|
||||
if err != nil {
|
||||
s.records <- Record{err: errors.Wrapf(err, "processing given header: %+v", s.Header)}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
for filename := range s.Files {
|
||||
s.processFile(filename)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) processHeader(header []string) (schema []idk.Field, err error) {
|
||||
schema = make([]idk.Field, len(header))
|
||||
for i, val := range header {
|
||||
schema[i], err = idk.HeaderToField(val, s.Log)
|
||||
if err != nil && s.JustDoIt {
|
||||
schema[i] = idk.StringField{NameVal: strings.ToLower(val)}
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return schema, nil
|
||||
}
|
||||
|
||||
func (s *Source) processFile(name string) {
|
||||
s.Log.Printf("processFile: %s", name)
|
||||
|
||||
f, err := openFileOrURL(name)
|
||||
if err != nil {
|
||||
s.records <- Record{err: errors.Wrapf(err, "opening %s", name)}
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
reader := csv.NewReader(f)
|
||||
reader.ReuseRecord = true
|
||||
reader.FieldsPerRecord = 0
|
||||
var nextErr error
|
||||
if s.expectHeader || s.IgnoreHeader {
|
||||
header, err := reader.Read()
|
||||
if err != nil {
|
||||
s.records <- Record{err: errors.Wrapf(err, "reading header from '%s'", name)}
|
||||
return
|
||||
}
|
||||
if s.expectHeader {
|
||||
newschema, err := s.processHeader(header)
|
||||
if err != nil {
|
||||
s.Log.Printf("processHeader error: %v\n", err)
|
||||
s.records <- Record{err: errors.Wrapf(err, "processing header from '%s': %+v", name, header)}
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(newschema, s.schema) {
|
||||
s.schema = newschema
|
||||
nextErr = idk.ErrSchemaChange
|
||||
}
|
||||
}
|
||||
}
|
||||
// weird hack to reuse memory rather than allocating eery
|
||||
// time. Would probably be better to get rid of the channel and
|
||||
// just the next CSV row on demand.
|
||||
recs := [2]Record{
|
||||
{data: make([]interface{}, len(s.schema)), err: nextErr},
|
||||
{data: make([]interface{}, len(s.schema))},
|
||||
}
|
||||
i := -1
|
||||
extraColumnsCount := 0
|
||||
row, err := reader.Read()
|
||||
for ; err == nil; row, err = reader.Read() {
|
||||
i += 1
|
||||
for j, val := range row {
|
||||
if len(s.schema) <= j {
|
||||
if extraColumnsCount == 0 {
|
||||
s.Log.Warnf("'%s': ignoring additional column(s) not included in the header specification", name)
|
||||
}
|
||||
extraColumnsCount++
|
||||
break
|
||||
}
|
||||
recs[i%2].data[j] = val
|
||||
}
|
||||
s.records <- recs[i%2]
|
||||
recs[i%2].err = nil
|
||||
}
|
||||
if extraColumnsCount > 0 {
|
||||
s.Log.Printf("Processing '%s': %d rows have more columns than header specification", name, extraColumnsCount)
|
||||
}
|
||||
if err != io.EOF {
|
||||
s.Log.Printf("ERROR Processing '%s': '%v'. Skipping rest of file.", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func NewSource() *Source {
|
||||
return &Source{
|
||||
records: make(chan Record, 0), // nolint: gosimple // do not change buffer size!
|
||||
}
|
||||
}
|
||||
|
||||
func openFileOrURL(name string) (io.ReadCloser, error) {
|
||||
var content io.ReadCloser
|
||||
if strings.HasPrefix(name, "http") {
|
||||
resp, err := http.Get(name)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getting via http")
|
||||
}
|
||||
if resp.StatusCode > 299 {
|
||||
return nil, errors.Errorf("got status %d via http.Get", resp.StatusCode)
|
||||
}
|
||||
content = resp.Body
|
||||
} else {
|
||||
f, err := os.Open(name)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "opening file")
|
||||
}
|
||||
content = f
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
func (s *Source) Close() error {
|
||||
return nil
|
||||
}
|
||||
39
idk/datagen/README.md
Normal file
39
idk/datagen/README.md
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
# Datagen Tool
|
||||
|
||||
## Help Usage
|
||||
|
||||
```sh
|
||||
Usage of datagen:
|
||||
-c, --concurrency int Number of concurrent sources and indexing routines to launch. (default 1)
|
||||
--dry-run Dry run - just flag parsing.
|
||||
-e, --end-at uint ID at which to stop generating records.
|
||||
--kafka.batch-size int Number of records to generate before sending them to Kafka all at once. Generally, larger means better throughput and more memory usage. (default 1000)
|
||||
--kafka.hosts strings Comma separated list of host:port pairs for Kafka. (default [])
|
||||
--kafka.registry-url string Location of Confluent Schema Registry. Must start with 'https://' if you want to use TLS.
|
||||
--kafka.subject string Kafka schema subject.
|
||||
--kafka.topic string Kafka topic to post to.
|
||||
--pilosa.batch-size int Number of records to read before indexing all of them at once. Generally, larger means better throughput and more memory usage. 1,048,576 might be a good number.
|
||||
--pilosa.cache-length uint Number of batches of ID mappings to cache. (default 64)
|
||||
--pilosa.hosts strings Comma separated list of host:port pairs for Pilosa. (default [])
|
||||
--pilosa.index string Name of Pilosa index.
|
||||
--seed int Seed to use for any random number generation.
|
||||
-s, --source string Source generator type. Running datagen with no arguments will list the available source types.
|
||||
-b, --start-from uint ID at which to start generating records.
|
||||
-t, --target string Destination for the generated data: [kafka, pilosa]. (default "pilosa")
|
||||
--track-progress Periodically print status updates on how many records have been sourced.
|
||||
```
|
||||
|
||||
## Example Usage
|
||||
|
||||
The following command will create 100 records in Pilosa index (starting at ID 0 and ending at ID 99)
|
||||
in the `equipment` index using the `equipment` data generator.
|
||||
|
||||
```sh
|
||||
datagen --source=equipment --pilosa.index=equipment --end-at=99
|
||||
```
|
||||
|
||||
## Adding New Sources
|
||||
|
||||
TODO: redo README (or delete?)
|
||||
|
||||
If you're looking to add a new Source to datagen, the best thing to do is use the special "custom" datagen source (`datagen --source=custom --custom-config=somefile.yaml`) and write a `somefile.yaml` which describes the data you want to generate. An example can be found in `datagen/testdata/custom.yaml`, and there are some more in the molecula/technical-validation repo.
|
||||
319
idk/datagen/all-field-types.go
Normal file
319
idk/datagen/all-field-types.go
Normal file
|
|
@ -0,0 +1,319 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
"github.com/molecula/featurebase/v3/pql"
|
||||
)
|
||||
|
||||
// Ensure AllFieldTypes implements interface.
|
||||
var _ Sourcer = (*AllFieldTypes)(nil)
|
||||
|
||||
// AllFieldTypes implements Sourcer, and returns a data
|
||||
// set containing one of every field type.
|
||||
type AllFieldTypes struct{}
|
||||
|
||||
// NewAllFieldTypes returns a new instance of AllFieldTypes.
|
||||
func NewAllFieldTypes(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &AllFieldTypes{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (a *AllFieldTypes) Source(cfg SourceConfig) idk.Source {
|
||||
epoch := time.Date(2010, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
layout := "2006-01-02"
|
||||
src := &AllFieldTypesSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
epoch: epoch,
|
||||
layout: layout,
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.BoolField{NameVal: "bool"}, // 1
|
||||
idk.DateIntField{
|
||||
NameVal: "dateint",
|
||||
Epoch: epoch,
|
||||
Unit: idk.Day,
|
||||
Layout: layout,
|
||||
}, // 2
|
||||
idk.DecimalField{NameVal: "decimal", Scale: 3}, // 3
|
||||
idk.IDArrayField{NameVal: "idarray"}, // 4
|
||||
idk.IDField{NameVal: "idfield"}, // 5
|
||||
//idk.IgnoreField{}, // TODO: maybe an "ignore" property instead?
|
||||
idk.IntField{
|
||||
NameVal: "int",
|
||||
Min: intptr(-100000),
|
||||
Max: intptr(200000),
|
||||
}, // 6 // TODO: foreignIndex
|
||||
idk.RecordTimeField{
|
||||
NameVal: "recordtime",
|
||||
Layout: layout,
|
||||
}, // 7
|
||||
idk.SignedIntBoolKeyField{NameVal: "signedintboolkey"}, // 8
|
||||
idk.StringArrayField{NameVal: "stringarray"}, // 9
|
||||
idk.StringField{NameVal: "string"}, // 10
|
||||
|
||||
// Include `mutex` fields.
|
||||
idk.IDField{NameVal: "idfieldmutex", Mutex: true}, // 11
|
||||
idk.StringField{NameVal: "stringmutex", Mutex: true}, // 12
|
||||
|
||||
// Include `quantum` fields.
|
||||
idk.IDArrayField{NameVal: "idarrayquantum", Quantum: "YMD", TTL: "0s"}, // 13
|
||||
idk.IDField{NameVal: "idfieldquantum", Quantum: "YMD", TTL: "0s"}, // 14
|
||||
idk.StringArrayField{NameVal: "stringarrayquantum", Quantum: "YMD", TTL: "0s"}, // 15
|
||||
idk.StringField{NameVal: "stringquantum", Quantum: "YMD", TTL: "0s"}, // 16
|
||||
idk.TimestampField{NameVal: "timestamp", Unit: idk.Second}, // 17
|
||||
},
|
||||
}
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.g = gen.New(gen.OptGenSeed(cfg.seed))
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (a *AllFieldTypes) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (a *AllFieldTypes) DefaultEndAt() uint64 {
|
||||
return 100
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (a *AllFieldTypes) Info() string {
|
||||
return "Generates data for all field types."
|
||||
}
|
||||
|
||||
// Ensure ExampleSource implements interface.
|
||||
var _ idk.Source = (*AllFieldTypesSource)(nil)
|
||||
|
||||
// AllFieldTypesSource is an instance of a source generated
|
||||
// by the Sourcer implementation AllFieldTypes.
|
||||
type AllFieldTypesSource struct {
|
||||
g *gen.Gen
|
||||
|
||||
cur uint64
|
||||
endAt uint64
|
||||
|
||||
epoch time.Time
|
||||
layout string
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
// Record implements idk.Source.
|
||||
func (a *AllFieldTypesSource) Record() (idk.Record, error) {
|
||||
if a.cur >= a.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
// Increment the ID.
|
||||
a.record[0] = a.cur
|
||||
|
||||
// 1 - Bool
|
||||
switch a.cur % 7 {
|
||||
case 0:
|
||||
a.record[1] = nil
|
||||
case 1:
|
||||
a.record[1] = true
|
||||
case 2:
|
||||
a.record[1] = false
|
||||
case 3:
|
||||
a.record[1] = "true"
|
||||
case 4:
|
||||
a.record[1] = "false"
|
||||
case 5:
|
||||
a.record[1] = 1
|
||||
case 6:
|
||||
a.record[1] = 0
|
||||
}
|
||||
|
||||
// 2 - DateIntField
|
||||
switch a.cur % 2 {
|
||||
case 0:
|
||||
a.record[2] = nil
|
||||
case 1:
|
||||
a.record[2] = a.epoch.AddDate(0, 0, int(a.cur)).Format(a.layout)
|
||||
}
|
||||
|
||||
// 3 - Decimal
|
||||
switch a.cur % 2 {
|
||||
case 0:
|
||||
a.record[3] = nil
|
||||
case 1:
|
||||
a.record[3] = pql.NewDecimal(int64(1003*a.cur), 2)
|
||||
}
|
||||
|
||||
// 4 - IDArray
|
||||
switch a.cur % 8 {
|
||||
case 7:
|
||||
a.record[4] = nil
|
||||
default:
|
||||
// Generate a random set field.
|
||||
set := a.g.Set(100, 1000, 10)
|
||||
vals := make([]uint64, 0, len(set))
|
||||
for v := range set {
|
||||
vals = append(vals, v)
|
||||
}
|
||||
a.record[4] = vals
|
||||
}
|
||||
|
||||
// 5 - ID (pilosa rowID field, not primary key)
|
||||
switch a.cur % 6 {
|
||||
case 5:
|
||||
a.record[5] = nil
|
||||
default:
|
||||
a.record[5] = a.g.R.Int63n(100000)
|
||||
}
|
||||
|
||||
// x - IgnoreField
|
||||
|
||||
// 6 - Int
|
||||
switch a.cur % 9 {
|
||||
case 8:
|
||||
a.record[6] = nil
|
||||
default:
|
||||
a.record[6] = a.g.R.Int63n(300000) - 100000
|
||||
}
|
||||
|
||||
// 7 - RecordTime
|
||||
switch a.cur % 7 {
|
||||
case 8:
|
||||
a.record[7] = nil
|
||||
default:
|
||||
a.record[7] = a.epoch.AddDate(5, 0, int(a.cur)).Format(a.layout)
|
||||
}
|
||||
|
||||
// 8 - SignedIntBoolKey
|
||||
switch a.cur % 10 {
|
||||
case 4:
|
||||
a.record[8] = nil
|
||||
default:
|
||||
a.record[8] = int64(2*(a.cur%2)-1) * a.g.R.Int63n(10000)
|
||||
}
|
||||
|
||||
// 9 - StringArray
|
||||
switch a.cur % 7 {
|
||||
case 5:
|
||||
a.record[9] = nil
|
||||
default:
|
||||
// Generate a random set field.
|
||||
set := a.g.Set(100, 1000, 10)
|
||||
vals := make([]string, 0, len(set))
|
||||
for v := range set {
|
||||
vals = append(vals, "v"+strconv.FormatUint(v, 10))
|
||||
}
|
||||
sort.Strings(vals)
|
||||
a.record[9] = vals
|
||||
}
|
||||
|
||||
// 10 - String
|
||||
switch a.cur % 8 {
|
||||
case 5:
|
||||
a.record[10] = nil
|
||||
default:
|
||||
rint := a.g.R.Int63n(2000)
|
||||
a.record[10] = "s" + strconv.FormatUint(uint64(rint), 10)
|
||||
}
|
||||
|
||||
// 11 - ID, mutex
|
||||
switch a.cur % 16 {
|
||||
case 6:
|
||||
a.record[11] = nil
|
||||
default:
|
||||
a.record[11] = a.g.R.Int63n(100000)
|
||||
}
|
||||
|
||||
// 12 - String, mutex
|
||||
switch a.cur % 17 {
|
||||
case 5:
|
||||
a.record[12] = nil
|
||||
default:
|
||||
rint := a.g.R.Int63n(2000)
|
||||
a.record[12] = "sm" + strconv.FormatUint(uint64(rint), 10)
|
||||
}
|
||||
|
||||
// 13 - IDArray
|
||||
switch a.cur % 9 {
|
||||
case 2:
|
||||
a.record[13] = nil
|
||||
default:
|
||||
// Generate a random set field.
|
||||
set := a.g.Set(100, 1000, 10)
|
||||
vals := make([]uint64, 0, len(set))
|
||||
for v := range set {
|
||||
vals = append(vals, v)
|
||||
}
|
||||
a.record[13] = vals
|
||||
}
|
||||
|
||||
// 14 - ID, quantum
|
||||
switch a.cur % 14 {
|
||||
case 7:
|
||||
a.record[14] = nil
|
||||
default:
|
||||
a.record[14] = a.g.R.Int63n(100000)
|
||||
}
|
||||
|
||||
// 15 - StringArray, quantum
|
||||
switch a.cur % 18 {
|
||||
case 7:
|
||||
a.record[15] = nil
|
||||
default:
|
||||
// Generate a random set field.
|
||||
set := a.g.Set(100, 1000, 10)
|
||||
vals := make([]string, 0, len(set))
|
||||
for v := range set {
|
||||
vals = append(vals, "sa"+strconv.FormatUint(v, 10))
|
||||
}
|
||||
sort.Strings(vals)
|
||||
a.record[15] = vals
|
||||
}
|
||||
|
||||
// 16 - String, quantum
|
||||
switch a.cur % 17 {
|
||||
case 5:
|
||||
a.record[16] = nil
|
||||
default:
|
||||
rint := a.g.R.Int63n(2000)
|
||||
a.record[16] = "sq" + strconv.FormatUint(uint64(rint), 10)
|
||||
}
|
||||
|
||||
// 17 - Timestamp
|
||||
switch a.cur % 15 {
|
||||
case 8:
|
||||
a.record[17] = nil
|
||||
default:
|
||||
rint := a.g.R.Int63n(604800) // seconds in a week
|
||||
a.record[17] = 1605041948 + rint // offset by sometime in 2021
|
||||
}
|
||||
|
||||
a.cur++
|
||||
|
||||
return a.record, nil
|
||||
}
|
||||
|
||||
// Schema implements idk.Source.
|
||||
func (a *AllFieldTypesSource) Schema() []idk.Field {
|
||||
return a.schema
|
||||
}
|
||||
|
||||
func (a *AllFieldTypesSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
311
idk/datagen/bank.go
Normal file
311
idk/datagen/bank.go
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
)
|
||||
|
||||
// Ensure Bank implements interface.
|
||||
var _ Sourcer = (*Bank)(nil)
|
||||
|
||||
// Bank implements Sourcer.
|
||||
type Bank struct {
|
||||
schema []idk.Field
|
||||
}
|
||||
|
||||
// NewBank returns a new instance of a Bank data generator.
|
||||
func NewBank(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Bank{
|
||||
schema: []idk.Field{
|
||||
idk.StringField{NameVal: "aba", Mutex: true},
|
||||
idk.StringField{NameVal: "db", Mutex: true},
|
||||
idk.IntField{NameVal: "user_id"},
|
||||
idk.StringField{NameVal: "custom_audiences"},
|
||||
// TODO generalize
|
||||
idk.DecimalField{NameVal: floatNames[0]},
|
||||
idk.BoolField{NameVal: boolNames[0]},
|
||||
idk.BoolField{NameVal: boolNames[1]},
|
||||
idk.StringField{NameVal: stringNames[0]},
|
||||
idk.IntField{NameVal: intNames[0]},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (b *Bank) Source(cfg SourceConfig) idk.Source {
|
||||
src := &BankSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
schema: b.schema,
|
||||
rand: rand.New(rand.NewSource(cfg.seed)),
|
||||
}
|
||||
|
||||
src.g = gen.New(gen.OptGenSeed(cfg.seed))
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
// TODO alloc?
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (b *Bank) PrimaryKeyFields() []string {
|
||||
return []string{"aba", "db", "user_id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (b *Bank) DefaultEndAt() uint64 {
|
||||
return 20000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (b *Bank) Info() string {
|
||||
return "Generates user data representative of a multi-site banking application"
|
||||
}
|
||||
|
||||
// Ensure BankSource implements interface.
|
||||
var _ idk.Source = (*BankSource)(nil)
|
||||
|
||||
// Ensure BankSource implements interface.
|
||||
//var _ idk.KafkaSource = (*BankSource)(nil) // TODO
|
||||
|
||||
// BankSource is a data generator which generates
|
||||
// data for all Pilosa field types.
|
||||
type BankSource struct {
|
||||
g *gen.Gen
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
|
||||
rand *rand.Rand
|
||||
}
|
||||
|
||||
var (
|
||||
// kafka cruft a.k.a. kruftka
|
||||
floatMeta = `"type": ["null", {"type": "float", "scale": 3}], "default": null`
|
||||
boolMeta = `"type": ["null", "boolean"], "default": null`
|
||||
stringMeta = `"type": ["null", "string"], "default": null, "mutex": true`
|
||||
intMeta = `"type": ["null", "int"], "default": null`
|
||||
|
||||
floatNames = []string{
|
||||
"pfm_category_total_current_balance__personal_loan",
|
||||
}
|
||||
|
||||
boolNames = []string{
|
||||
"pfm_boolean__personal_loan",
|
||||
"pfm_boolean__mortgage",
|
||||
}
|
||||
|
||||
stringNames = []string{
|
||||
"survey_5dfe1a89_29fa_4505_ac00_4cd055b758ab",
|
||||
}
|
||||
|
||||
intNames = []string{
|
||||
"product_recency__consumer_loan",
|
||||
}
|
||||
|
||||
floatNullChance = 0.1
|
||||
boolNullChance = 0.1
|
||||
stringNullChance = 0.1
|
||||
intNullChance = 0.1
|
||||
|
||||
// TODO these are effectively consts right now
|
||||
floatCount = 1
|
||||
boolCount = 2
|
||||
stringCount = 1
|
||||
intCount = 1
|
||||
)
|
||||
|
||||
// Record implements idk.Source.
|
||||
func (b *BankSource) Record() (idk.Record, error) {
|
||||
if b.cur >= b.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
// Increment the ID.
|
||||
//b.record[0] = b.cur
|
||||
b.cur++
|
||||
|
||||
b.record[0] = b.ABA()
|
||||
b.record[1] = b.Db()
|
||||
b.record[2] = b.UserID()
|
||||
b.record[3] = b.CustomAudiences()
|
||||
|
||||
// TODO generalize
|
||||
if floatNullChance < b.rand.Float64() {
|
||||
b.record[4] = b.rand.Float64()
|
||||
}
|
||||
// TODO: handle null chance
|
||||
|
||||
if boolNullChance < b.rand.Float64() {
|
||||
b.record[5] = b.rand.Intn(2) == 1
|
||||
}
|
||||
// TODO: handle null chance
|
||||
|
||||
if boolNullChance < b.rand.Float64() {
|
||||
b.record[6] = b.rand.Intn(2) == 1
|
||||
}
|
||||
// TODO: handle null chance
|
||||
|
||||
if stringNullChance < b.rand.Float64() {
|
||||
b.record[7] = text(b.rand, 1, 6, true, true, true, false)
|
||||
}
|
||||
// TODO: handle null chance
|
||||
|
||||
if intNullChance < b.rand.Float64() {
|
||||
b.record[8] = int64(b.rand.Intn(1000000))
|
||||
}
|
||||
// TODO: handle null chance
|
||||
|
||||
return b.record, nil
|
||||
}
|
||||
|
||||
// KafkaRecord might implement idk.KafkaSource
|
||||
// TODO: going to want to generate the kafka record from the idk record
|
||||
// automatic handling would be nice, but allowing per-source is probably important,
|
||||
// e.g. to handle special cases for individual kafka environments
|
||||
func (b *BankSource) KafkaRecord() string {
|
||||
fields := []string{
|
||||
`"aba": "` + b.ABA() + `"`,
|
||||
`"db": "` + b.Db() + `"`,
|
||||
`"user_id": ` + strconv.FormatInt(int64(b.UserID()), 10),
|
||||
`"custom_audiences": {"string": "` + b.CustomAudiences() + `"}`,
|
||||
}
|
||||
|
||||
for n := 0; n < floatCount; n++ {
|
||||
if floatNullChance < b.rand.Float64() {
|
||||
floatStr := strconv.FormatFloat(b.rand.Float64(), 'f', 3, 64)
|
||||
fields = append(fields, `"float_`+floatNames[n]+`": {"float": `+floatStr+`}`)
|
||||
}
|
||||
}
|
||||
for n := 0; n < boolCount; n++ {
|
||||
if boolNullChance < b.rand.Float64() {
|
||||
boolStr := strconv.FormatBool(b.rand.Intn(2) == 1)
|
||||
fields = append(fields, `"bool_`+boolNames[n]+`": {"boolean": `+boolStr+`}`)
|
||||
}
|
||||
}
|
||||
|
||||
for n := 0; n < stringCount; n++ {
|
||||
if stringNullChance < b.rand.Float64() {
|
||||
stringStr := text(b.rand, 1, 6, true, true, true, false)
|
||||
fields = append(fields, `"string_`+stringNames[n]+`": {"string": "`+stringStr+`"}`)
|
||||
}
|
||||
}
|
||||
|
||||
for n := 0; n < intCount; n++ {
|
||||
if intNullChance < b.rand.Float64() {
|
||||
intStr := strconv.FormatInt(int64(b.rand.Intn(1000000)), 10)
|
||||
fields = append(fields, `"int_`+intNames[n]+`": {"int": `+intStr+`}`)
|
||||
}
|
||||
}
|
||||
|
||||
return `{` + strings.Join(fields, ", ") + `}`
|
||||
}
|
||||
|
||||
// ABA returns a random 9 numeric digit string with about 27000 possible values.
|
||||
func (b *BankSource) ABA() string {
|
||||
num := b.rand.Intn(27000) + 22213
|
||||
num2 := num/10 - 1213
|
||||
numstr := strconv.Itoa(num)
|
||||
num2str := strconv.Itoa(num2)
|
||||
numstrbytes := append([]byte(numstr), num2str[3], numstr[0], numstr[1], numstr[2])
|
||||
return string(numstrbytes)
|
||||
}
|
||||
|
||||
// Db returns a db
|
||||
func (b *BankSource) Db() string {
|
||||
return text(b.rand, 1, 6, true, true, true, false)
|
||||
}
|
||||
|
||||
// UserID returns a user ID
|
||||
func (b *BankSource) UserID() int {
|
||||
return b.rand.Intn(10000000) // 10 mil
|
||||
}
|
||||
|
||||
// CustomAudiences returns a fake Custom Audience string
|
||||
func (b *BankSource) CustomAudiences() string {
|
||||
return text(b.rand, 1, 6, true, true, true, false)
|
||||
}
|
||||
|
||||
// TODO: move to gen.go
|
||||
var lowerLetters = []rune("abcdefghijklmnopqrstuvwxyz")
|
||||
var upperLetters = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
||||
var numeric = []rune("0123456789")
|
||||
var specialChars = []rune(`!'@#$%^&*()_+-=[]{};:",./?`)
|
||||
|
||||
func text(rand *rand.Rand, atLeast, atMost int, allowLower, allowUpper, allowNumeric, allowSpecial bool) string {
|
||||
allowedChars := []rune{}
|
||||
if allowLower {
|
||||
allowedChars = append(allowedChars, lowerLetters...)
|
||||
}
|
||||
if allowUpper {
|
||||
allowedChars = append(allowedChars, upperLetters...)
|
||||
}
|
||||
if allowNumeric {
|
||||
allowedChars = append(allowedChars, numeric...)
|
||||
}
|
||||
if allowSpecial {
|
||||
allowedChars = append(allowedChars, specialChars...)
|
||||
}
|
||||
|
||||
result := []rune{}
|
||||
nTimes := rand.Intn(atMost-atLeast+1) + atLeast
|
||||
for i := 0; i < nTimes; i++ {
|
||||
result = append(result, allowedChars[rand.Intn(len(allowedChars))])
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
// Schema implements idk.Source
|
||||
func (b *BankSource) Schema() []idk.Field {
|
||||
return b.schema
|
||||
}
|
||||
|
||||
// KafkaSchema implements idk.KafkaSource
|
||||
func (b *BankSource) KafkaSchema() string {
|
||||
meta := `"type": "record",
|
||||
"name": "bank_user_data",
|
||||
"namespace": "bank.user.data",
|
||||
"doc": "Per-user bank data"`
|
||||
|
||||
fields := []string{
|
||||
`{"name": "aba", "type": "string"}`,
|
||||
`{"name": "db", "type": "string"}`,
|
||||
`{"name": "user_id", "type": "int"}`,
|
||||
`{"name": "custom_audiences", "type": ["null", "string"], "default": null}`,
|
||||
}
|
||||
|
||||
// TODO redmove loops if Schema() is not generalized
|
||||
for n := 0; n < floatCount; n++ {
|
||||
fields = append(fields, `{"name": "float_`+floatNames[n]+`", `+floatMeta+`}`)
|
||||
}
|
||||
for n := 0; n < boolCount; n++ {
|
||||
fields = append(fields, `{"name": "bool_`+boolNames[n]+`", `+boolMeta+`}`)
|
||||
}
|
||||
for n := 0; n < stringCount; n++ {
|
||||
fields = append(fields, `{"name": "string_`+stringNames[n]+`", `+stringMeta+`}`)
|
||||
}
|
||||
for n := 0; n < intCount; n++ {
|
||||
fields = append(fields, `{"name": "int_`+intNames[n]+`", `+intMeta+`}`)
|
||||
}
|
||||
schema := fmt.Sprintf(`{%s, "fields": [%s]}`, meta, strings.Join(fields, ", "))
|
||||
|
||||
return schema
|
||||
}
|
||||
|
||||
func (b *BankSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
128
idk/datagen/claim.go
Normal file
128
idk/datagen/claim.go
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Claim implements interface.
|
||||
var _ Sourcer = (*Claim)(nil)
|
||||
|
||||
// Claim implements Sourcer.
|
||||
type Claim struct{}
|
||||
|
||||
// NewClaim returns a new instance of Claim.
|
||||
func NewClaim(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Claim{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (c *Claim) Source(cfg SourceConfig) idk.Source {
|
||||
min := int64(0)
|
||||
src := &ClaimSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
rand: rand.New(rand.NewSource(19)),
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"},
|
||||
idk.StringField{NameVal: "service_type"},
|
||||
idk.IntField{NameVal: "date"},
|
||||
idk.IntField{NameVal: "item_id", Min: &min},
|
||||
idk.StringArrayField{NameVal: "part"},
|
||||
},
|
||||
}
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
|
||||
src.partZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(parts)-1))
|
||||
|
||||
src.record[4] = make([]string, 4)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (c *Claim) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (c *Claim) DefaultEndAt() uint64 {
|
||||
return 50000000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (c *Claim) Info() string {
|
||||
return "TODO"
|
||||
}
|
||||
|
||||
// Ensure ClaimSource implements interface.
|
||||
var _ idk.Source = (*ClaimSource)(nil)
|
||||
|
||||
type ClaimSource struct {
|
||||
Log logger.Logger
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
rand *rand.Rand
|
||||
partZipf *rand.Zipf
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
var parts = []string{"cpu", "ram", "motherboard", "cooler", "case", "speaker", "microphone", "nic", "wlan", "screen", "keyboard", "disk", "ssd", "optical", "fan", "powersupp", "part0", "part1", "part2", "part3", "part4", "part5", "part6", "part7", "part8", "part9", "part10", "part11", "part12", "part13", "part14", "part15", "part16", "part17", "part18", "part19", "part20", "part21", "part22", "part23", "part24", "part25", "part26", "part27", "part28", "part29", "part30", "part31", "part32", "part33", "part34", "part35", "part36", "part37", "part38", "part39", "part40", "part41", "part42", "part43", "part44", "part45", "part46", "part47", "part48", "part49", "part50", "part51", "part52", "part53", "part54", "part55", "part56", "part57", "part58", "part59", "part60", "part61", "part62", "part63", "part64", "part65", "part66", "part67", "part68", "part69", "part70", "part71", "part72", "part73", "part74", "part75", "part76", "part77", "part78", "part79", "part80", "part81", "part82", "part83", "part84"}
|
||||
var claimTypes = []string{"warranty", "damage", "spill", "intentional", "accidental", "collateral", "priority", "replacement", "reimbursement"}
|
||||
|
||||
func (s *ClaimSource) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.cur
|
||||
s.record[1] = claimTypes[s.rand.Intn(len(claimTypes))]
|
||||
s.record[2] = int64(time.Duration(s.rand.Intn(int(currentDur))) / (time.Hour * 24)) // day
|
||||
s.record[3] = s.rand.Intn(1500000000)
|
||||
s.record[4] = s.record[4].([]string)[:randNumParts(s.rand)]
|
||||
for i := range s.record[4].([]string) {
|
||||
s.record[4].([]string)[i] = parts[s.partZipf.Uint64()]
|
||||
}
|
||||
s.cur++
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *ClaimSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *ClaimSource) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*ClaimSource)(nil)
|
||||
|
||||
func randNumParts(r *rand.Rand) int {
|
||||
n := r.Intn(100)
|
||||
if n < 80 {
|
||||
return 1
|
||||
}
|
||||
if n < 90 {
|
||||
return 2
|
||||
}
|
||||
if n < 95 {
|
||||
return 3
|
||||
}
|
||||
return 4
|
||||
}
|
||||
|
||||
func (s *ClaimSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
607
idk/datagen/cmd.go
Normal file
607
idk/datagen/cmd.go
Normal file
|
|
@ -0,0 +1,607 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/glycerine/vprint"
|
||||
pilosaclient "github.com/molecula/featurebase/v3/client"
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk/common"
|
||||
"github.com/molecula/featurebase/v3/idk/kafka"
|
||||
)
|
||||
|
||||
const (
|
||||
TargetFeaturebase = "featurebase"
|
||||
TargetKafka = "kafka"
|
||||
TargetKafkaStatic = "kafkastatic"
|
||||
)
|
||||
|
||||
// Main is the top-level datagen struct. It represents datagen-specific
|
||||
// configuration parameters, as well as sub-level parameteres specific
|
||||
// to each target.
|
||||
type Main struct {
|
||||
idkMain *idk.Main
|
||||
|
||||
KafkaPut *kafka.PutSource `flag:"-"`
|
||||
|
||||
srcs []idk.Source
|
||||
srcLock sync.Mutex
|
||||
|
||||
cfg SourceGeneratorConfig
|
||||
|
||||
Source string `short:"s" flag:"source" help:"Source generator type. Running datagen with no arguments will list the available source types."`
|
||||
Target string `short:"t" flag:"target" help:"Destination for the generated data: [kafka, featurebase]."`
|
||||
|
||||
Concurrency int `short:"c" flag:"concurrency" help:"Number of concurrent sources and indexing routines to launch."`
|
||||
|
||||
StartFrom uint64 `short:"b" help:"ID at which to start generating records."`
|
||||
EndAt uint64 `short:"e" help:"ID at which to stop generating records."`
|
||||
|
||||
Seed int64 `short:"" help:"Seed to use for any random number generation."`
|
||||
|
||||
TrackProgress bool `short:"" help:"Periodically print status updates on how many records have been sourced."`
|
||||
UseIngestAPI bool `help:"Experimental: use new HTTP/JSON ingest API instead of low-level import API. Probably slow, does not support packed bools."`
|
||||
|
||||
UseShardTransactionalEndpoint bool `flag:"use-shard-transactional-endpoint" help:"Use experimental transactional endpoint"`
|
||||
|
||||
CustomConfig string `short:"" help:"File from which to pull configuration for 'custom' source."`
|
||||
|
||||
// Used strictly for configuration of the targets.
|
||||
Pilosa PilosaConfig
|
||||
Kafka KafkaConfig
|
||||
|
||||
DryRun bool `help:"Dry run - just flag parsing."`
|
||||
|
||||
AuthToken string `flag:"auth-token" help:"Authentication token for FeatureBase"`
|
||||
Verbose bool `flag:"verbose" help:"Enable extended logging for debug"`
|
||||
Datadog bool `flag:"datadog" help:"Enable datadog profiling"`
|
||||
}
|
||||
|
||||
// PilosaConfig is meant to represent the scoped (pilosa.*) configuration options
|
||||
// to be used when target = pilosa. These are really just a sub-set of idk.Main,
|
||||
// containing only those arguments that really apply to datagen.
|
||||
type PilosaConfig struct {
|
||||
Hosts []string `short:"" help:"Comma separated list of host:port pairs for FeatureBase."`
|
||||
Index string `short:"" help:"Name of FeatureBase index."`
|
||||
BatchSize int `short:"" help:"Number of records to read before indexing all of them at once. Generally, larger means better throughput and more memory usage. 1,048,576 might be a good number."`
|
||||
CacheLength uint64 `help:"Number of batches of ID mappings to cache."`
|
||||
}
|
||||
|
||||
// KafkaConfig is meant to represent the scoped (pilosa.*) configuration options
|
||||
// to be used when target = kafka. These are really just a sub-set of kafka.PutSource,
|
||||
// containing only those arguments that really apply to datagen.
|
||||
type KafkaConfig struct {
|
||||
idk.ConfluentCommand
|
||||
Topic string `short:"" help:"Kafka topic to post to."`
|
||||
Subject string `short:"" help:"Kafka schema subject."`
|
||||
BatchSize int `short:"" help:"Number of records to generate before sending them to Kafka all at once. Generally, larger means better throughput and more memory usage."`
|
||||
ReplicationFactor int `short:"" help:"set replication factor for kafka cluster"`
|
||||
NumPartitions int `short:"" help:"set partition for kafka cluster"`
|
||||
}
|
||||
|
||||
// NewMain returns a new instance of Main.
|
||||
func NewMain() *Main {
|
||||
return &Main{
|
||||
idkMain: &idk.Main{},
|
||||
KafkaPut: &kafka.PutSource{},
|
||||
Target: TargetFeaturebase,
|
||||
|
||||
Concurrency: 1,
|
||||
Pilosa: PilosaConfig{
|
||||
CacheLength: 64,
|
||||
},
|
||||
Kafka: KafkaConfig{
|
||||
BatchSize: 1000,
|
||||
ReplicationFactor: 1,
|
||||
NumPartitions: 1,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Sourcer is an interface describing a type which can generate sources,
|
||||
// which in this case are `idk.Source`. It contains a few additional
|
||||
// methods which are used to configure the source.
|
||||
type Sourcer interface {
|
||||
Source(SourceConfig) idk.Source
|
||||
PrimaryKeyFields() []string
|
||||
DefaultEndAt() uint64
|
||||
Info() string
|
||||
}
|
||||
|
||||
// SourceConfig is the configuration required by a Sourcer when creating
|
||||
// a new Source.
|
||||
type SourceConfig struct {
|
||||
startFrom uint64
|
||||
endAt uint64
|
||||
total uint64
|
||||
seed int64
|
||||
}
|
||||
|
||||
// sourcerTypes is a map of all available custom Sourcer
|
||||
// implementations.
|
||||
var sourcerTypes = map[string]func(SourceGeneratorConfig) Sourcer{
|
||||
"all-field-types": NewAllFieldTypes,
|
||||
"bank": NewBank,
|
||||
"claim": NewClaim,
|
||||
"customer_segmentation": NewPerson,
|
||||
"customer_segmentation_linkedin": NewLinkedInPerson,
|
||||
"customer": NewCustomer,
|
||||
"custom": NewCustom,
|
||||
"dell": NewDell,
|
||||
"dwarranty": NewDWarranty,
|
||||
"example": NewExample,
|
||||
"hughes": NewHughes,
|
||||
"equipment": NewEquipment,
|
||||
"network_ts": NewNetwork,
|
||||
"power_ts": NewTimeseries,
|
||||
"site": NewSite,
|
||||
"transactions_ts": NewTransaction,
|
||||
"item": NewItem,
|
||||
"kitchensink": NewKitchenSink,
|
||||
"kitchensink_keyed": NewKitchenSinkKeyed,
|
||||
"merck": NewMerck,
|
||||
"paloalto": NewPaloAlto,
|
||||
"power_scenario_1": NewPower1,
|
||||
"power_scenario_1_2": NewPower1_2,
|
||||
"sizing": NewSizing,
|
||||
"stringpk": NewStringPK,
|
||||
"texas_health": NewTexasHealth,
|
||||
"transactions_scenario_1": NewTransaction1,
|
||||
"warranty": NewWarranty,
|
||||
}
|
||||
|
||||
var sourcerTypeGroups = map[string][]string{
|
||||
"iot": {
|
||||
"equipment",
|
||||
"network_ts",
|
||||
"power_ts",
|
||||
"site",
|
||||
"transactions_ts",
|
||||
},
|
||||
}
|
||||
|
||||
func (m *Main) info() string {
|
||||
// Sort the sourcerTypes map by key.
|
||||
keys := make([]string, 0, len(sourcerTypes))
|
||||
for k := range sourcerTypes {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
ret := "\n------------------------------------------\n"
|
||||
ret += "A source must be specified with --source\n"
|
||||
ret += "The following source types are supported:\n\n"
|
||||
|
||||
alreadyPrinted := make(map[string]struct{})
|
||||
|
||||
for group, keys := range sourcerTypeGroups {
|
||||
ret += fmt.Sprintf(" %s\n", group)
|
||||
for _, k := range keys {
|
||||
info, _ := typeInfo(k)
|
||||
ret += fmt.Sprintf(" %-21s: %s\n", k, info)
|
||||
alreadyPrinted[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
for _, k := range keys {
|
||||
if _, ok := alreadyPrinted[k]; ok {
|
||||
continue
|
||||
}
|
||||
info, _ := typeInfo(k)
|
||||
ret += fmt.Sprintf(" %-25s: %s\n", k, info)
|
||||
}
|
||||
ret += "\n"
|
||||
return ret
|
||||
}
|
||||
|
||||
func typeInfo(key string) (string, error) {
|
||||
srcrFn, ok := sourcerTypes[key]
|
||||
if !ok {
|
||||
return "", errors.Errorf("invalid key: %s", key)
|
||||
}
|
||||
src := srcrFn(SourceGeneratorConfig{})
|
||||
return src.Info(), nil
|
||||
}
|
||||
|
||||
// Preload configures the Sources based on the concurrency
|
||||
// and start/end range.
|
||||
func (m *Main) Preload() error {
|
||||
if m.Source == "" {
|
||||
return errors.New(m.info())
|
||||
}
|
||||
|
||||
cfg := SourceGeneratorConfig{
|
||||
StartFrom: m.StartFrom,
|
||||
EndAt: m.EndAt,
|
||||
Concurrency: m.Concurrency,
|
||||
Seed: m.Seed,
|
||||
CustomConfig: m.CustomConfig,
|
||||
}
|
||||
|
||||
pks, srcs, err := m.makeSources(m.Source, cfg)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "getting sources")
|
||||
}
|
||||
m.idkMain = idk.NewMain()
|
||||
// TODO: this is a little hacky because of the complexity around
|
||||
// PrimaryKeyFields and IDField, and the fact that those are
|
||||
// currently being set before having a schema.
|
||||
// See: https://github.com/molecula/featurebase/v3/idk/issues/147
|
||||
switch {
|
||||
case len(pks) == 1 && pks[0] == "id":
|
||||
m.idkMain.IDField = "id"
|
||||
case len(pks) > 0:
|
||||
m.idkMain.IDField = ""
|
||||
m.idkMain.PrimaryKeyFields = pks
|
||||
default:
|
||||
m.idkMain.AutoGenerate = true
|
||||
m.idkMain.ExternalGenerate = true
|
||||
}
|
||||
switch m.Target {
|
||||
case TargetKafka, TargetKafkaStatic:
|
||||
m.KafkaPut, _ = kafka.NewPutSource()
|
||||
m.KafkaPut.Concurrency = m.Concurrency
|
||||
m.KafkaPut.NewSource = m.newSource
|
||||
m.KafkaPut.TrackProgress = m.TrackProgress
|
||||
m.KafkaPut.BatchSize = m.Kafka.BatchSize
|
||||
m.KafkaPut.ConfluentCommand = m.Kafka.ConfluentCommand
|
||||
m.KafkaPut.ConfigMap, err = common.SetupConfluent(&m.Kafka.ConfluentCommand)
|
||||
m.KafkaPut.Target = m.Target
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "setup confluent")
|
||||
}
|
||||
if m.Kafka.Topic != "" {
|
||||
m.KafkaPut.Topic = m.Kafka.Topic
|
||||
}
|
||||
if m.Kafka.Subject != "" {
|
||||
m.KafkaPut.Subject = m.Kafka.Subject
|
||||
}
|
||||
if m.Verbose {
|
||||
m.KafkaPut.Log = logger.NewVerboseLogger(os.Stderr)
|
||||
}
|
||||
m.KafkaPut.ReplicationFactor = m.Kafka.ReplicationFactor
|
||||
m.KafkaPut.NumPartitions = m.Kafka.NumPartitions
|
||||
m.KafkaPut.FBPrimaryKeyFields = m.idkMain.PrimaryKeyFields
|
||||
m.KafkaPut.FBIDField = m.idkMain.IDField
|
||||
m.KafkaPut.FBIndexName = m.Pilosa.Index
|
||||
|
||||
default:
|
||||
m.idkMain.Namespace = "ingester_datagen"
|
||||
m.idkMain.Concurrency = m.Concurrency
|
||||
m.idkMain.CacheLength = m.Pilosa.CacheLength
|
||||
m.idkMain.NewSource = m.newSource
|
||||
m.idkMain.TrackProgress = m.TrackProgress
|
||||
m.idkMain.UseIngestAPI = m.UseIngestAPI
|
||||
m.idkMain.AuthToken = m.AuthToken
|
||||
m.idkMain.UseShardTransactionalEndpoint = m.UseShardTransactionalEndpoint
|
||||
if len(m.Pilosa.Hosts) > 0 {
|
||||
m.idkMain.PilosaHosts = m.Pilosa.Hosts
|
||||
}
|
||||
m.idkMain.Index = m.Pilosa.Index
|
||||
if m.Pilosa.BatchSize > 0 {
|
||||
m.idkMain.BatchSize = m.Pilosa.BatchSize
|
||||
}
|
||||
}
|
||||
|
||||
m.srcs = srcs
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sources returns the list of sources. This is typically
|
||||
// called by tests, after the generator has created the sources
|
||||
// during Preload().
|
||||
func (m *Main) Sources() []idk.Source {
|
||||
return m.srcs
|
||||
}
|
||||
|
||||
// Run generates data for the specified target.
|
||||
func (m *Main) Run() error {
|
||||
switch m.Target {
|
||||
case TargetKafka, TargetKafkaStatic:
|
||||
return m.KafkaPut.Run()
|
||||
default:
|
||||
return m.idkMain.Run()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Main) PrintPlan() {
|
||||
cfg := m.cfg
|
||||
startFrom := cfg.StartFrom
|
||||
endAt := cfg.EndAt - 1 // adjust for the implementation detail `endAt++` in generator.Sources
|
||||
concurrency := cfg.Concurrency
|
||||
RecordCount := endAt - startFrom
|
||||
|
||||
switch m.Target {
|
||||
case TargetKafka:
|
||||
Topic := m.Kafka.Topic
|
||||
NumPartitions := m.Kafka.NumPartitions
|
||||
ReplicationFactor := m.Kafka.ReplicationFactor
|
||||
Subject := m.Kafka.Subject
|
||||
Hosts := m.KafkaPut.KafkaBootstrapServers
|
||||
RegURL := m.KafkaPut.SchemaRegistryURL
|
||||
if m.Kafka.Topic == "" {
|
||||
Topic = m.KafkaPut.Topic
|
||||
}
|
||||
if m.Kafka.Subject == "" {
|
||||
Subject = m.KafkaPut.Subject
|
||||
}
|
||||
fmt.Printf(`Datagen config:
|
||||
hosts: %s
|
||||
start id: %s
|
||||
end id: %s
|
||||
total generated: %s
|
||||
concurrency: %d
|
||||
registry URL: %s
|
||||
topic: %s
|
||||
subject: %s
|
||||
num partitions: %d
|
||||
replication factor: %d
|
||||
`,
|
||||
strings.Join(Hosts, ", "),
|
||||
AddThousandSep(startFrom),
|
||||
AddThousandSep(endAt),
|
||||
AddThousandSep(RecordCount),
|
||||
concurrency,
|
||||
RegURL,
|
||||
Topic,
|
||||
Subject,
|
||||
NumPartitions,
|
||||
ReplicationFactor,
|
||||
)
|
||||
case TargetKafkaStatic:
|
||||
Topic := m.Kafka.Topic
|
||||
NumPartitions := m.Kafka.NumPartitions
|
||||
ReplicationFactor := m.Kafka.ReplicationFactor
|
||||
Subject := m.Kafka.Subject
|
||||
Hosts := m.KafkaPut.KafkaBootstrapServers
|
||||
if m.Kafka.Topic == "" {
|
||||
Topic = m.KafkaPut.Topic
|
||||
}
|
||||
if m.Kafka.Subject == "" {
|
||||
Subject = m.KafkaPut.Subject
|
||||
}
|
||||
fmt.Printf(`Datagen config:
|
||||
hosts: %s
|
||||
start id: %s
|
||||
end id: %s
|
||||
total generated: %s
|
||||
concurrency: %d
|
||||
topic: %s
|
||||
subject: %s
|
||||
num partitions: %d
|
||||
replication factor: %d
|
||||
`,
|
||||
strings.Join(Hosts, ", "),
|
||||
AddThousandSep(startFrom),
|
||||
AddThousandSep(endAt),
|
||||
AddThousandSep(RecordCount),
|
||||
concurrency,
|
||||
Topic,
|
||||
Subject,
|
||||
NumPartitions,
|
||||
ReplicationFactor,
|
||||
)
|
||||
default:
|
||||
Hosts := m.Pilosa.Hosts
|
||||
BatchSize := m.Pilosa.BatchSize
|
||||
if len(m.Pilosa.Hosts) <= 0 {
|
||||
Hosts = m.idkMain.PilosaHosts
|
||||
}
|
||||
if m.Pilosa.BatchSize <= 0 {
|
||||
BatchSize = m.idkMain.BatchSize
|
||||
}
|
||||
BatchCount := uint64(math.Ceil(float64(RecordCount) / float64(BatchSize)))
|
||||
|
||||
if m.Pilosa.Index == "" {
|
||||
m.Pilosa.Index = "(not specified)"
|
||||
}
|
||||
fmt.Printf(`Datagen config:
|
||||
hosts: %s
|
||||
index: %s
|
||||
start id: %s
|
||||
end id: %s
|
||||
total generated: %s
|
||||
concurrency: %d
|
||||
batch size: %s
|
||||
total batches: %s
|
||||
`,
|
||||
strings.Join(Hosts, ", "),
|
||||
m.Pilosa.Index,
|
||||
AddThousandSep(startFrom),
|
||||
AddThousandSep(endAt),
|
||||
AddThousandSep(RecordCount),
|
||||
concurrency,
|
||||
AddThousandSep(uint64(BatchSize)),
|
||||
AddThousandSep(BatchCount),
|
||||
)
|
||||
|
||||
fmt.Println("Schema:")
|
||||
for _, field := range m.srcs[0].Schema() {
|
||||
fmt.Printf("%T %+[1]v\n", field)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func AddThousandSep(num uint64) string {
|
||||
in := strconv.FormatUint(num, 10)
|
||||
out := ""
|
||||
for i := 1; i <= len(in); i++ {
|
||||
out = string(in[len(in)-i]) + out
|
||||
if i != 0 && i != len(in) && i%3 == 0 {
|
||||
out = "," + out
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// newSource pops the next idk.Source from srcs.
|
||||
func (m *Main) newSource() (source idk.Source, err error) {
|
||||
m.srcLock.Lock()
|
||||
defer m.srcLock.Unlock()
|
||||
|
||||
if len(m.srcs) < 1 {
|
||||
return nil, errors.New("requested too many sources")
|
||||
}
|
||||
source = m.srcs[0]
|
||||
m.srcs = m.srcs[1:]
|
||||
return source, nil
|
||||
}
|
||||
|
||||
// PilosaClient is a helper for tests. The datagen command
|
||||
// used to embed idk.Main, so datagen tests could call
|
||||
// Main.PilosaClient(), but idk.Main is no longer
|
||||
// directly embedded.
|
||||
func (m *Main) PilosaClient() *pilosaclient.Client {
|
||||
if m.idkMain == nil {
|
||||
return nil
|
||||
}
|
||||
return m.idkMain.PilosaClient()
|
||||
}
|
||||
|
||||
// NoStats is a helper for tests.
|
||||
func (m *Main) NoStats() {
|
||||
if m.idkMain != nil {
|
||||
m.idkMain.Stats = ""
|
||||
}
|
||||
}
|
||||
|
||||
// Sources implements the SourceGenerator interface.
|
||||
func (m *Main) makeSources(key string, cfg SourceGeneratorConfig) ([]string, []idk.Source, error) {
|
||||
srcrFn, ok := sourcerTypes[key]
|
||||
if !ok {
|
||||
return nil, nil, errors.Errorf("invalid key: %s", key)
|
||||
}
|
||||
srcr := srcrFn(cfg)
|
||||
|
||||
// Get the primary key fields for the source.
|
||||
pks := srcr.PrimaryKeyFields()
|
||||
|
||||
startFrom, endAt := cfg.StartFrom, cfg.EndAt
|
||||
|
||||
if endAt == 0 {
|
||||
if dflt := srcr.DefaultEndAt(); dflt > 0 {
|
||||
endAt = dflt
|
||||
} else {
|
||||
endAt = startFrom + 99 // Default to 100 records.
|
||||
}
|
||||
}
|
||||
if startFrom > endAt {
|
||||
return nil, nil, errors.Errorf("invalid start/end: %d/%d", startFrom, endAt)
|
||||
}
|
||||
endAt++ // endAt is used as `a.cur >= a.endAt`, so increment it to get the expected, inclusive, behavior.
|
||||
|
||||
// Calculate the concurrency, total, and start/end boundaries
|
||||
// based on the configuration values.
|
||||
m.cfg = SourceGeneratorConfig{
|
||||
StartFrom: startFrom,
|
||||
EndAt: endAt,
|
||||
Concurrency: cfg.Concurrency,
|
||||
Seed: cfg.Seed,
|
||||
CustomConfig: cfg.CustomConfig,
|
||||
}
|
||||
concurrency, startEnds, total := startEnds(m.cfg)
|
||||
|
||||
seedGen := rand.New(rand.NewSource(cfg.Seed))
|
||||
|
||||
srcs := make([]idk.Source, concurrency)
|
||||
|
||||
// Distribute the total range of records over a number of
|
||||
// Sources equal to the concurrency.
|
||||
for i := 0; i < concurrency; i++ {
|
||||
srcCfg := SourceConfig{
|
||||
startFrom: startEnds[i].start,
|
||||
endAt: startEnds[i].end,
|
||||
total: total,
|
||||
seed: cfg.Seed,
|
||||
}
|
||||
vprint.VV("source config for i: %d, %#v", i, srcCfg)
|
||||
|
||||
src := srcr.Source(srcCfg)
|
||||
|
||||
// TODO: pull seed from config?
|
||||
if s, ok := src.(Seedable); ok {
|
||||
s.Seed(seedGen.Int63())
|
||||
}
|
||||
|
||||
srcs[i] = src
|
||||
}
|
||||
|
||||
return pks, srcs, nil
|
||||
}
|
||||
|
||||
// startEnds returns the start/end boundaries for each source
|
||||
// run needed based on cfg.StartFrom, cfg.EndAt, and cfg.Concurrency.
|
||||
// It also returns the calculated total, and the final concurrency
|
||||
// (in case it had to be adjusted).
|
||||
// returns (concurrency, startEnds, total)
|
||||
func startEnds(cfg SourceGeneratorConfig) (int, []startEnd, uint64) {
|
||||
var startFrom uint64 = cfg.StartFrom
|
||||
var endAt uint64 = cfg.EndAt
|
||||
var concurrency int = cfg.Concurrency
|
||||
|
||||
if concurrency == 0 {
|
||||
return 0, []startEnd{}, 0
|
||||
}
|
||||
|
||||
total := endAt - startFrom
|
||||
if uint64(concurrency) > total {
|
||||
concurrency = int(total)
|
||||
}
|
||||
|
||||
var current uint64
|
||||
|
||||
interval := total / uint64(concurrency)
|
||||
if startFrom > current {
|
||||
current = startFrom
|
||||
}
|
||||
leftover := total - (interval * uint64(concurrency))
|
||||
|
||||
ses := make([]startEnd, concurrency)
|
||||
|
||||
// Distribute the total range of records over a number of
|
||||
// Sources equal to the concurrency.
|
||||
for i := 0; i < concurrency; i++ {
|
||||
var start uint64 = current
|
||||
var end uint64
|
||||
current += interval
|
||||
// Include one of the leftovers in each iteration until there
|
||||
// are no more.
|
||||
if uint64(i) < leftover {
|
||||
current++
|
||||
}
|
||||
// start is inclusive, end is exclusive
|
||||
end = current
|
||||
|
||||
ses[i] = startEnd{
|
||||
start: start,
|
||||
end: end,
|
||||
}
|
||||
}
|
||||
|
||||
return concurrency, ses, total
|
||||
}
|
||||
|
||||
type record []interface{}
|
||||
|
||||
func (r record) Commit(ctx context.Context) error { return nil }
|
||||
|
||||
func (r record) Data() []interface{} {
|
||||
return r
|
||||
}
|
||||
|
||||
type startEnd struct {
|
||||
start uint64
|
||||
end uint64
|
||||
}
|
||||
34
idk/datagen/common.go
Normal file
34
idk/datagen/common.go
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
)
|
||||
|
||||
// SourceGenerator is an interface for anything which can generate
|
||||
// data by providing one or more idk.Source. It also contains
|
||||
// a method for getting information about the supported sources.
|
||||
// Info() - return information about the generator along with
|
||||
// a list of supported types
|
||||
// Sources() - provided a string key and configuration, returns
|
||||
// a list of primary key fields and a list of sources.
|
||||
type SourceGenerator interface {
|
||||
Info() string
|
||||
Sources(string, SourceGeneratorConfig) ([]string, []idk.Source, error)
|
||||
Config() SourceGeneratorConfig
|
||||
}
|
||||
|
||||
// SourceGeneratorConfig provides configuration values used by
|
||||
// implementations of the SourceGenerator interface.
|
||||
type SourceGeneratorConfig struct {
|
||||
StartFrom uint64
|
||||
EndAt uint64
|
||||
Concurrency int
|
||||
Seed int64
|
||||
CustomConfig string
|
||||
}
|
||||
|
||||
// Seedable is an interface representing anything for which
|
||||
// a seed can be provided.
|
||||
type Seedable interface {
|
||||
Seed(int64)
|
||||
}
|
||||
949
idk/datagen/custom.go
Normal file
949
idk/datagen/custom.go
Normal file
|
|
@ -0,0 +1,949 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/pkg/errors"
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
// Ensure Custom implements Sourcer interface.
|
||||
var _ Sourcer = (*Custom)(nil)
|
||||
|
||||
// Custom implements Sourcer
|
||||
type Custom struct {
|
||||
err error
|
||||
CustomConfig *CustomConfig
|
||||
IDKAndGenFields *IDKAndGenFields
|
||||
}
|
||||
|
||||
// NewCustom returns a new instance of Custom.
|
||||
func NewCustom(cfg SourceGeneratorConfig) Sourcer {
|
||||
conf := &CustomConfig{}
|
||||
if bytes, err := ioutil.ReadFile(cfg.CustomConfig); err != nil {
|
||||
return &Custom{err: errors.Wrap(err, "reading custom config file")}
|
||||
} else if err = yaml.Unmarshal(bytes, conf); err != nil {
|
||||
return &Custom{err: errors.Wrap(err, "unmarshaling custom config file")}
|
||||
} else if err := conf.PostUnmarshal(); err != nil {
|
||||
return &Custom{err: errors.Wrap(err, "post unmarshal")}
|
||||
} else if ig, err := conf.GetIDKFields(); err != nil {
|
||||
return &Custom{err: errors.Wrap(err, "getting IDK fields from custom config file")}
|
||||
} else {
|
||||
return &Custom{
|
||||
CustomConfig: conf,
|
||||
IDKAndGenFields: ig,
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// CustomConfig represents the JSON/yaml configuration for the "custom" datagen source.
|
||||
type CustomConfig struct {
|
||||
// Fields describe the data to be generated.
|
||||
Fields []*GenField `json:"fields"`
|
||||
|
||||
// IDKParams configure how the generated data should be treated by
|
||||
// the IDK (and probably ultimately ingested into FeatureBase).
|
||||
IDKParams IDKParams `json:"idk_params"`
|
||||
}
|
||||
|
||||
// GenField describes one field of the data to be generated. Many of
|
||||
// the struct fields are only used for certain values of the 'Type'
|
||||
// field. See the commented custom.yaml example as a reference.
|
||||
//
|
||||
// GenField says nothing about what should be done with the generated
|
||||
// data, or how it might map to FeatureBase fields.
|
||||
//
|
||||
// NOTE: If you add fields here, make sure you add them to the custom
|
||||
// UnmarshalJSON method below.
|
||||
type GenField struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Distribution string `json:"distribution"`
|
||||
Min int64 `json:"min"`
|
||||
Max int64 `json:"max"`
|
||||
MinFloat float64 `json:"min_float"`
|
||||
MaxFloat float64 `json:"max_float"`
|
||||
Repeat bool `json:"repeat"`
|
||||
Step int64 `json:"step"`
|
||||
S float64 `json:"s"`
|
||||
V float64 `json:"v"`
|
||||
SourceFile string `json:"source_file"`
|
||||
GeneratorType string `json:"generator_type"`
|
||||
Cardinality uint64 `json:"cardinality"`
|
||||
Charset string `json:"charset"`
|
||||
MinLen uint64 `json:"min_len"`
|
||||
MaxLen uint64 `json:"max_len"`
|
||||
MinNum uint64 `json:"min_num"`
|
||||
MaxNum uint64 `json:"max_num"`
|
||||
MinStepDuration string `json:"min_step_duration"`
|
||||
MaxStepDuration string `json:"max_step_duration"`
|
||||
minStepDuration time.Duration
|
||||
maxStepDuration time.Duration
|
||||
MinDate time.Time `json:"min_date"`
|
||||
MaxDate time.Time `json:"max_date"`
|
||||
TimeFormat TimeFormat `json:"time_format"`
|
||||
TimeUnit idk.Unit `json:"time_unit"`
|
||||
NullChance float64 `json:"null_chance"` // between 0 and 1.0. Fraction of the time that we get a null value for this field
|
||||
}
|
||||
|
||||
// PostUnmarshal does post-processing of the CustomConfig object after
|
||||
// the basic JSON/yaml unmarshal, like converting time strings to
|
||||
// durations. There's probably an elegant way to do this during the
|
||||
// Unmarshal, but I couldn't figure it out.
|
||||
func (c *CustomConfig) PostUnmarshal() error {
|
||||
for i, g := range c.Fields {
|
||||
if g.MinStepDuration != "" {
|
||||
dur, err := time.ParseDuration(g.MinStepDuration)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "parsing step_duration_min")
|
||||
}
|
||||
c.Fields[i].minStepDuration = dur
|
||||
}
|
||||
if g.MaxStepDuration != "" {
|
||||
dur, err := time.ParseDuration(g.MaxStepDuration)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "parsing step_duration_max")
|
||||
}
|
||||
c.Fields[i].maxStepDuration = dur
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultIDKField returns default IDK configuration for a given
|
||||
// GenField. This is convenient for the common case of generated data
|
||||
// wanting to be ingested into FeatureBase so that the user doesn't
|
||||
// have to specify explicit "idk_params" configuration for every
|
||||
// field.
|
||||
func (g *GenField) DefaultIDKField() (idk.Field, error) {
|
||||
switch g.Type {
|
||||
case "uint":
|
||||
return idk.IDField{NameVal: g.Name}, nil
|
||||
case "int":
|
||||
var min int64 = g.Min
|
||||
var max int64 = g.Max
|
||||
return idk.IntField{NameVal: g.Name, Min: &min, Max: &max}, nil
|
||||
case "string":
|
||||
return idk.StringField{NameVal: g.Name}, nil
|
||||
case "string-set":
|
||||
return idk.StringArrayField{NameVal: g.Name}, nil
|
||||
case "uint-set":
|
||||
return idk.IDArrayField{NameVal: g.Name}, nil
|
||||
case "timestamp":
|
||||
return idk.TimestampField{NameVal: g.Name}, nil
|
||||
case "float":
|
||||
return idk.DecimalField{NameVal: g.Name, Scale: 2}, nil
|
||||
default:
|
||||
return nil, errors.Errorf("no default IDK field for type: '%s'", g.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// IDKParams describes how data from CustomConfig.Fields should map to IDK fields.
|
||||
type IDKParams struct {
|
||||
Fields map[string][]IngestField `json:"fields"`
|
||||
PrimaryKeyConfig PrimaryKeyConfig `json:"primary_key_config"`
|
||||
}
|
||||
|
||||
// IngestField is a json/yaml configuration for an IDK Field. See the
|
||||
// ToIDKField method for how this configuration maps to each possible
|
||||
// IDK Field.
|
||||
type IngestField struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Layout string `json:"layout"`
|
||||
Epoch time.Time `json:"epoch"`
|
||||
Unit string `json:"unit"`
|
||||
Keyed bool `json:"keyed"`
|
||||
Mutex bool `json:"mutex"`
|
||||
TimeQuantum string `json:"time_quantum"`
|
||||
Min *int64 `json:"min"`
|
||||
Max *int64 `json:"max"`
|
||||
Scale int64 `json:"scale"`
|
||||
ForeignIndex string `json:"foreign_index"`
|
||||
Granularity string `json:"granularity"`
|
||||
TTL string `json:"ttl"`
|
||||
}
|
||||
|
||||
// ToIDKField converts the general IngestField to a specific IDK Field.
|
||||
func (f IngestField) ToIDKField(g *GenField) (idk.Field, error) {
|
||||
name := g.Name
|
||||
destName := g.Name
|
||||
if f.Name != "" {
|
||||
destName = f.Name
|
||||
}
|
||||
switch f.Type {
|
||||
case "ID":
|
||||
return idk.IDField{NameVal: name, DestNameVal: destName, Quantum: f.TimeQuantum, Mutex: f.Mutex, TTL: f.TTL}, nil
|
||||
case "IDArray":
|
||||
return idk.IDArrayField{NameVal: name, DestNameVal: destName, Quantum: f.TimeQuantum, TTL: f.TTL}, nil
|
||||
case "Int":
|
||||
return idk.IntField{NameVal: name, DestNameVal: destName, Min: f.Min, Max: f.Max, ForeignIndex: f.ForeignIndex}, nil
|
||||
case "RecordTime":
|
||||
return idk.RecordTimeField{NameVal: name, DestNameVal: destName, Layout: f.Layout, Epoch: f.Epoch, Unit: idk.Unit(f.Unit)}, nil
|
||||
case "String":
|
||||
return idk.StringField{NameVal: name, DestNameVal: destName, Mutex: f.Mutex, Quantum: f.TimeQuantum, TTL: f.TTL}, nil
|
||||
case "StringArray":
|
||||
return idk.StringArrayField{NameVal: name, DestNameVal: destName, Quantum: f.TimeQuantum, TTL: f.TTL}, nil
|
||||
case "Timestamp":
|
||||
return idk.TimestampField{NameVal: name, DestNameVal: destName, Layout: f.Layout, Epoch: f.Epoch, Unit: idk.Unit(f.Unit), Granularity: f.Granularity}, nil
|
||||
case "Decimal":
|
||||
return idk.DecimalField{NameVal: name, DestNameVal: destName, Scale: f.Scale}, nil
|
||||
default:
|
||||
return nil, errors.Errorf("unsupported idk field type: '%s' for field named: '%s'", f.Type, f.Name)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// PrimaryKeyConfig specifies what should be interpreted as the record
|
||||
// identifier in Featurebase.
|
||||
type PrimaryKeyConfig struct {
|
||||
Field string `json:"field"`
|
||||
}
|
||||
|
||||
// IDKAndGenFields keeps the IDK schema and generated data in
|
||||
// sync. schema and genFields will always be the same length, and the
|
||||
// idk.Fields in schema will always be non-nil. Items in genFields may
|
||||
// be nil which indicates that the corresponding idk.Field in schema
|
||||
// will use data generated by the last non-nil genField.
|
||||
// e.g.
|
||||
//
|
||||
// schema: IDField, RecordTimeField, TimestampField
|
||||
// genFields: Gen{"id"}, Gen{"timestamp"}, nil
|
||||
//
|
||||
// The TimestampField will get values from Gen{"timestamp"} since its
|
||||
// corresponding genField is nil.
|
||||
type IDKAndGenFields struct {
|
||||
schema []idk.Field
|
||||
genFields []*GenField
|
||||
}
|
||||
|
||||
// Append adds to IDKAndGenFields and ensures that schema and genFields stay in sync.
|
||||
func (ig *IDKAndGenFields) Append(f idk.Field, g *GenField) {
|
||||
ig.schema = append(ig.schema, f)
|
||||
ig.genFields = append(ig.genFields, g)
|
||||
}
|
||||
|
||||
// GetIDKFields figures out what IDK Fields should be created based on
|
||||
// the generated data fields and the IDK configuration parameters. If
|
||||
// multiple IDK fields are generated for a single genField, the
|
||||
// corresponding entries beyond the first in IDKAndGenFields.genFields
|
||||
// will be nil. We'll only create one generator and use the value it
|
||||
// generates for all those fields in the generated record.
|
||||
func (cc *CustomConfig) GetIDKFields() (*IDKAndGenFields, error) {
|
||||
ig := &IDKAndGenFields{}
|
||||
for _, genField := range cc.Fields {
|
||||
idkFieldConfig, ok := cc.IDKParams.Fields[genField.Name]
|
||||
if !ok {
|
||||
idkField, err := genField.DefaultIDKField()
|
||||
if err != nil {
|
||||
return ig, errors.Wrapf(err, "getting default IDK field for %+v", genField)
|
||||
}
|
||||
ig.Append(idkField, genField)
|
||||
continue
|
||||
}
|
||||
for i, ingestField := range idkFieldConfig {
|
||||
idkField, err := ingestField.ToIDKField(genField)
|
||||
if err != nil {
|
||||
return ig, errors.Wrap(err, "getting configured IDK field")
|
||||
}
|
||||
if i == 0 {
|
||||
ig.Append(idkField, genField)
|
||||
} else {
|
||||
ig.Append(idkField, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ig, nil
|
||||
}
|
||||
|
||||
// Source returns a configured Custom idk.Source. It does not
|
||||
// currently have explicit support for concurrency or paritioned data
|
||||
// generation as some of the other datagen sources do.
|
||||
func (c *Custom) Source(cfg SourceConfig) idk.Source {
|
||||
if c.err != nil {
|
||||
log.Fatal(c.err) // TODO add error return arg to Sourcer interface
|
||||
}
|
||||
|
||||
r := rand.New(rand.NewSource(cfg.seed))
|
||||
|
||||
generators := make([]FieldGenerator, len(c.IDKAndGenFields.genFields))
|
||||
for i, g := range c.IDKAndGenFields.genFields {
|
||||
var err error
|
||||
if g == nil {
|
||||
generators[i] = nil
|
||||
continue
|
||||
}
|
||||
generators[i], err = g.Generator(r)
|
||||
log.Printf("Generator for %s, %+v\n", g.Name, generators[i])
|
||||
if err != nil {
|
||||
log.Fatal(errors.Wrap(err, "getting generators"))
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("schema: %+v", c.IDKAndGenFields.schema)
|
||||
recordsToGenerate := uint64(0)
|
||||
if cfg.endAt > 0 {
|
||||
recordsToGenerate = cfg.endAt - cfg.startFrom
|
||||
}
|
||||
return &CustomSource{
|
||||
conf: c.CustomConfig,
|
||||
schema: c.IDKAndGenFields.schema,
|
||||
|
||||
generators: generators,
|
||||
record: make([]interface{}, len(generators)),
|
||||
recordsToGenerate: recordsToGenerate,
|
||||
}
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (c *Custom) PrimaryKeyFields() []string {
|
||||
if c.err != nil {
|
||||
log.Fatal(errors.Wrap(c.err, "can't get primary key fields"))
|
||||
}
|
||||
return []string{c.CustomConfig.IDKParams.PrimaryKeyConfig.Field}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface. Not used for Custom.
|
||||
func (_ *Custom) DefaultEndAt() uint64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (_ *Custom) Info() string {
|
||||
return "Generates data from a yaml definition file."
|
||||
}
|
||||
|
||||
// Ensure CustomSource implements interface.
|
||||
var _ idk.Source = (*CustomSource)(nil)
|
||||
|
||||
// CustomSource is an instance of a source generated
|
||||
// by the Sourcer implementation Custom.
|
||||
type CustomSource struct {
|
||||
conf *CustomConfig
|
||||
generators []FieldGenerator
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
recordsToGenerate uint64
|
||||
recordCounter uint64
|
||||
}
|
||||
|
||||
// Schema returns the IDK fields which were determined from the data
|
||||
// generation and ingest configuration parameters in the custom yaml
|
||||
// file.
|
||||
func (cs *CustomSource) Schema() []idk.Field {
|
||||
return cs.schema
|
||||
}
|
||||
|
||||
// Record uses the configured generators to emit a custom record. If
|
||||
// any of the generators are nil, it uses the value from the last
|
||||
// non-nil generator.
|
||||
func (cs *CustomSource) Record() (idk.Record, error) {
|
||||
// track last generated value
|
||||
var last interface{}
|
||||
for i, gen := range cs.generators {
|
||||
// this indicates that the same value is being used for multiple IDK fields.
|
||||
if gen == nil {
|
||||
cs.record[i] = last
|
||||
continue
|
||||
}
|
||||
var err error
|
||||
cs.record[i], err = gen.Generate(cs.record[i])
|
||||
if err == io.EOF {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "generating for %+v", cs.schema[i])
|
||||
}
|
||||
last = cs.record[i]
|
||||
}
|
||||
if cs.recordsToGenerate > 0 && cs.recordCounter >= cs.recordsToGenerate {
|
||||
//break when number records produced
|
||||
return nil, io.EOF
|
||||
} else {
|
||||
cs.recordCounter++
|
||||
}
|
||||
return cs.record, nil
|
||||
}
|
||||
|
||||
func (c *CustomSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// FieldGenerator is an interface for generating values for a particular field.
|
||||
type FieldGenerator interface {
|
||||
// Generate produces a value. It takes in the previous value as an
|
||||
// optimization—some implementations may opt to reuse a slice, for
|
||||
// example.
|
||||
Generate(oldval interface{}) (interface{}, error)
|
||||
}
|
||||
|
||||
type NullChanceGenerator struct {
|
||||
R *rand.Rand
|
||||
NullChance float64
|
||||
G FieldGenerator
|
||||
}
|
||||
|
||||
func (g *NullChanceGenerator) Generate(oldval interface{}) (interface{}, error) {
|
||||
if g.R.Float64() < g.NullChance {
|
||||
return nil, nil
|
||||
}
|
||||
return g.G.Generate(oldval)
|
||||
}
|
||||
|
||||
// Generator creates a data generator based on the GenField.
|
||||
func (g *GenField) Generator(r *rand.Rand) (fg FieldGenerator, err error) {
|
||||
if g.NullChance < 0 || g.NullChance > 1.0 {
|
||||
return nil, errors.Errorf("null_chance must be in the range [0.0, 1.0], but got: %f", g.NullChance)
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if g.NullChance == 0.0 {
|
||||
return
|
||||
}
|
||||
// if NullChance > 0, we'll wrap the generator we're going to return in a NullChanceGenerator
|
||||
fg = &NullChanceGenerator{
|
||||
R: r,
|
||||
NullChance: g.NullChance,
|
||||
G: fg,
|
||||
}
|
||||
}()
|
||||
switch g.Type {
|
||||
case "uint":
|
||||
if g.Distribution != "sequential" && g.Distribution != "" {
|
||||
return nil, errors.Errorf("unsupported distribution %s for uint field", g.Distribution)
|
||||
}
|
||||
if g.Step < 1 {
|
||||
return nil, errors.Errorf("step must be >=1 for uint, got: %d", g.Step)
|
||||
}
|
||||
return &SequentialUintFieldGenerator{
|
||||
cur: uint64(g.Min),
|
||||
Min: uint64(g.Min),
|
||||
Max: uint64(g.Max),
|
||||
Repeat: g.Repeat,
|
||||
Step: uint64(g.Step),
|
||||
}, nil
|
||||
case "int":
|
||||
return getIntGenerator(g, r)
|
||||
case "string":
|
||||
return getStringGenerator(g, r)
|
||||
case "string-set":
|
||||
sg, err := getStringGenerator(g, r)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getting string generator")
|
||||
}
|
||||
return NewStringSetGenerator(g, r, sg)
|
||||
case "uint-set":
|
||||
ig, err := getIntGenerator(g, r)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "getting int generator")
|
||||
}
|
||||
return &UintSetGenerator{
|
||||
G: ig,
|
||||
R: r,
|
||||
|
||||
MinNum: g.MinNum,
|
||||
MaxNum: g.MaxNum,
|
||||
}, nil
|
||||
case "timestamp":
|
||||
return getTimestampGenerator(g, r)
|
||||
case "float":
|
||||
switch g.Distribution {
|
||||
case "uniform", "":
|
||||
return &UniformFloatGenerator{
|
||||
R: r,
|
||||
MinFloat: g.MinFloat,
|
||||
MaxFloat: g.MaxFloat,
|
||||
}, nil
|
||||
default:
|
||||
return nil, errors.Errorf("unsupported distribution: '%s'", g.Distribution)
|
||||
}
|
||||
default:
|
||||
return nil, errors.Errorf("unsupported type '%s'", g.Type)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func getIntGenerator(g *GenField, r *rand.Rand) (FieldGenerator, error) {
|
||||
if g.Max == 0 && g.Min == 0 {
|
||||
g.Max = 1000 // default
|
||||
}
|
||||
if g.Max <= g.Min {
|
||||
return nil, errors.Errorf("max must be > min for int field min: %d, max: %d", g.Min, g.Max)
|
||||
}
|
||||
switch g.Distribution {
|
||||
case "uniform", "":
|
||||
return &UniformIntFieldGenerator{
|
||||
R: r,
|
||||
Min: g.Min,
|
||||
Max: g.Max,
|
||||
}, nil
|
||||
case "zipfian":
|
||||
z := rand.NewZipf(r, g.S, g.V, uint64(g.Max-g.Min))
|
||||
if z == nil {
|
||||
return nil, errors.Errorf("err getting int zipf: s: %f, v: %f, imax: %d", g.S, g.V, uint64(g.Max-g.Min))
|
||||
}
|
||||
return &ZipfianIntFieldGenerator{
|
||||
Z: z,
|
||||
Min: g.Min,
|
||||
}, nil
|
||||
default:
|
||||
return nil, errors.Errorf("unsupported distribution for int, got: %s", g.Distribution)
|
||||
}
|
||||
}
|
||||
|
||||
func getStringGenerator(g *GenField, r *rand.Rand) (FieldGenerator, error) {
|
||||
if g.SourceFile != "" {
|
||||
vals, err := readLines(g.SourceFile)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "reading source file for field '%s'", g.Name)
|
||||
}
|
||||
switch g.Distribution {
|
||||
case "zipfian", "":
|
||||
if g.S == 0.0 && g.V == 0.0 {
|
||||
g.S, g.V = 1.1, 5.1 // sane defaults
|
||||
}
|
||||
z := rand.NewZipf(r, g.S, g.V, uint64(len(vals)-1))
|
||||
if z == nil {
|
||||
return nil, errors.Errorf("err getting string zipf: s: %f, v: %f, imax: %d", g.S, g.V, uint64(len(vals)-1))
|
||||
}
|
||||
return &ZipfianStringSourceFieldGenerator{
|
||||
Source: vals,
|
||||
Z: z,
|
||||
}, nil
|
||||
case "uniform":
|
||||
return &UniformStringSourceFieldGenerator{
|
||||
Source: vals,
|
||||
R: r,
|
||||
}, nil
|
||||
case "fixed": // use probabilities from file
|
||||
vals, probs, err := readLinesWithProbabilities(g.SourceFile)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "reading source file for field '%s'", g.Name)
|
||||
}
|
||||
return NewFixedProbabilitySourceFieldGenerator(vals, probs, r)
|
||||
default:
|
||||
return nil, errors.Errorf("unknown distribution: '%s'", g.Distribution)
|
||||
}
|
||||
} else if g.GeneratorType == "random-string" {
|
||||
if g.MinLen > g.MaxLen || g.MaxLen == 0 {
|
||||
return nil, errors.Errorf("invalid max and min lengths: min(<=max): %d, max(>0): %d", g.MinLen, g.MaxLen)
|
||||
}
|
||||
return &RandomStringGenerator{
|
||||
R: r,
|
||||
MinLen: g.MinLen,
|
||||
MaxLen: g.MaxLen,
|
||||
Charset: g.Charset,
|
||||
}, nil
|
||||
} else if g.Distribution == "shifting" {
|
||||
return NewShiftingStringGenerator(g, r)
|
||||
}
|
||||
return nil, errors.Errorf("unsupported string config for field '%s'", g.Name)
|
||||
}
|
||||
|
||||
func getTimestampGenerator(g *GenField, r *rand.Rand) (FieldGenerator, error) {
|
||||
switch g.Distribution {
|
||||
case "increasing":
|
||||
if g.TimeFormat == "" {
|
||||
g.TimeFormat = timestampFormat
|
||||
}
|
||||
if !(g.TimeFormat == unixFormat || g.TimeFormat == timestampFormat) {
|
||||
return nil, errors.Errorf("'%s' is not a valid output format", g.TimeFormat)
|
||||
}
|
||||
return &IncreasingTimestampGenerator{
|
||||
R: r,
|
||||
MinDate: g.MinDate,
|
||||
MaxDate: g.MaxDate,
|
||||
MaxStepDuration: g.maxStepDuration,
|
||||
MinStepDuration: g.minStepDuration,
|
||||
Repeat: g.Repeat,
|
||||
OutputFormat: g.TimeFormat,
|
||||
Unit: g.TimeUnit,
|
||||
}, nil
|
||||
default:
|
||||
return nil, errors.Errorf("unimplemented distribution: %s", g.Distribution)
|
||||
}
|
||||
}
|
||||
|
||||
// SequentialUintFieldGenerator generates unsigned integers
|
||||
// sequentially from a minimum up to a maximum with a configurable
|
||||
// step. It will start from the beginning when it reaches the max if
|
||||
// repeat==true.
|
||||
type SequentialUintFieldGenerator struct {
|
||||
cur uint64
|
||||
Min uint64
|
||||
Max uint64
|
||||
Repeat bool
|
||||
Step uint64
|
||||
}
|
||||
|
||||
func (g *SequentialUintFieldGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
ret := g.cur
|
||||
g.cur += g.Step
|
||||
if g.cur > g.Max {
|
||||
if g.Repeat {
|
||||
g.cur = g.Min
|
||||
} else {
|
||||
return nil, io.EOF
|
||||
}
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// UniformIntFieldGenerator generates random integers between Min and Max with a uniform distribution
|
||||
type UniformIntFieldGenerator struct {
|
||||
R *rand.Rand
|
||||
Min int64
|
||||
Max int64
|
||||
}
|
||||
|
||||
func (g *UniformIntFieldGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
return g.R.Int63n(g.Max+1-g.Min) + g.Min, nil
|
||||
}
|
||||
|
||||
type ZipfianIntFieldGenerator struct {
|
||||
Z *rand.Zipf
|
||||
Min int64
|
||||
}
|
||||
|
||||
func (g *ZipfianIntFieldGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
return int64(g.Z.Uint64()) + g.Min, nil
|
||||
}
|
||||
|
||||
type ZipfianStringSourceFieldGenerator struct {
|
||||
Source []string
|
||||
Z *rand.Zipf
|
||||
}
|
||||
|
||||
func (g *ZipfianStringSourceFieldGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
return g.Source[g.Z.Uint64()], nil
|
||||
}
|
||||
|
||||
type UniformStringSourceFieldGenerator struct {
|
||||
Source []string
|
||||
R *rand.Rand
|
||||
}
|
||||
|
||||
func (g *UniformStringSourceFieldGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
return g.Source[g.R.Intn(len(g.Source))], nil
|
||||
}
|
||||
|
||||
func readLines(filename string) ([]string, error) {
|
||||
file, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer file.Close()
|
||||
|
||||
out := make([]string, 0)
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
out = append(out, scanner.Text())
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func readLinesWithProbabilities(filename string) ([]string, []float64, error) {
|
||||
file, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
values := make([]string, 0)
|
||||
probabilities := make([]float64, 0)
|
||||
r := csv.NewReader(file)
|
||||
r.FieldsPerRecord = -1
|
||||
r.ReuseRecord = true
|
||||
|
||||
var record []string
|
||||
for record, err = r.Read(); err == nil; record, err = r.Read() {
|
||||
if len(record) == 0 {
|
||||
continue
|
||||
}
|
||||
values = append(values, record[0])
|
||||
if len(record) > 1 {
|
||||
var p float64
|
||||
p, err = strconv.ParseFloat(record[1], 64)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrapf(err, "could not parse '%s' as a probability", record[1])
|
||||
}
|
||||
probabilities = append(probabilities, p)
|
||||
} else {
|
||||
probabilities = append(probabilities, -1.0)
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
}
|
||||
return values, probabilities, err
|
||||
}
|
||||
|
||||
type RandomStringGenerator struct {
|
||||
R *rand.Rand
|
||||
|
||||
MinLen uint64
|
||||
MaxLen uint64
|
||||
Charset string
|
||||
}
|
||||
|
||||
func (g *RandomStringGenerator) Generate(prev interface{}) (interface{}, error) {
|
||||
length := g.R.Int63n(int64(g.MaxLen-g.MinLen)+1) + int64(g.MinLen)
|
||||
buf, ok := prev.([]byte)
|
||||
if !ok {
|
||||
buf = make([]byte, g.MaxLen)
|
||||
}
|
||||
buf = buf[:0]
|
||||
for i := int64(0); i < length; i++ {
|
||||
buf = append(buf, g.Charset[g.R.Intn(len(g.Charset))])
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
type StringSetGenerator struct {
|
||||
G FieldGenerator // for generating individual strings
|
||||
R *rand.Rand
|
||||
|
||||
MinNum uint64
|
||||
MaxNum uint64
|
||||
}
|
||||
|
||||
func NewStringSetGenerator(g *GenField, r *rand.Rand, fg FieldGenerator) (*StringSetGenerator, error) {
|
||||
return &StringSetGenerator{
|
||||
G: fg,
|
||||
R: r,
|
||||
|
||||
MinNum: g.MinNum,
|
||||
MaxNum: g.MaxNum,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *StringSetGenerator) Generate(prev interface{}) (interface{}, error) {
|
||||
sslice, ok := prev.([]string)
|
||||
if !ok {
|
||||
sslice = make([]string, g.MaxNum)
|
||||
}
|
||||
sslice = sslice[:0]
|
||||
num := g.R.Int63n(int64(g.MaxNum-g.MinNum)+1) + int64(g.MinNum)
|
||||
var val []byte
|
||||
for i := int64(0); i < num; i++ {
|
||||
vali, err := g.G.Generate(val)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "generating string")
|
||||
}
|
||||
if val, ok := vali.([]byte); ok {
|
||||
sslice = append(sslice, string(val))
|
||||
} else {
|
||||
sslice = append(sslice, vali.(string))
|
||||
}
|
||||
}
|
||||
return sslice, nil
|
||||
}
|
||||
|
||||
type UintSetGenerator struct {
|
||||
G FieldGenerator
|
||||
R *rand.Rand
|
||||
|
||||
MinNum uint64
|
||||
MaxNum uint64
|
||||
}
|
||||
|
||||
func (g *UintSetGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
num := g.R.Intn(int(g.MaxNum-g.MinNum)+1) + int(g.MinNum)
|
||||
// ok, so we allocate a new slice for each record here, gross, I
|
||||
// know. Problem is that IDK ultimately passes these to
|
||||
// client.Batch.Add which adds the slice directly to a thing. This
|
||||
// works differently for the string-set case (with []string) which
|
||||
// is kind of horrible, and we should really make Batch.Add very
|
||||
// clear and consistent about its semantics. TODO Right now
|
||||
// though, we have this function which allocates a new slice every
|
||||
// time, and the string ones above reuse the previous slice, and I
|
||||
// just wanted to make sure everyone knows that that's on purpose.
|
||||
uslice := make([]uint64, num)
|
||||
for i := 0; i < num; i++ {
|
||||
val, err := g.G.Generate(nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "generating uint")
|
||||
}
|
||||
uslice[i] = uint64((val.(int64)))
|
||||
}
|
||||
return uslice, nil
|
||||
}
|
||||
|
||||
type IncreasingTimestampGenerator struct {
|
||||
R *rand.Rand
|
||||
prev time.Time
|
||||
|
||||
MinDate time.Time
|
||||
MaxDate time.Time
|
||||
MinStepDuration time.Duration
|
||||
MaxStepDuration time.Duration
|
||||
Repeat bool
|
||||
|
||||
OutputFormat TimeFormat // "time", "int"
|
||||
Unit idk.Unit // "s", "ms", "us", "ns"
|
||||
}
|
||||
|
||||
type TimeFormat string
|
||||
|
||||
const (
|
||||
timestampFormat = "timestamp" // time.Time
|
||||
unixFormat = "unix" // unix epoch number (int64)
|
||||
)
|
||||
|
||||
func (g *IncreasingTimestampGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
step := time.Duration(g.R.Int63n(int64(g.MaxStepDuration)-int64(g.MinStepDuration)+1)) + g.MinStepDuration
|
||||
|
||||
if g.prev.IsZero() {
|
||||
g.prev = g.MinDate
|
||||
}
|
||||
|
||||
next := g.prev.Add(step)
|
||||
if next.After(g.MaxDate) {
|
||||
if !g.Repeat {
|
||||
return nil, io.EOF
|
||||
}
|
||||
next = g.MinDate.Add(step)
|
||||
}
|
||||
g.prev = next
|
||||
var ret interface{} = next
|
||||
if g.OutputFormat == unixFormat {
|
||||
unitDuration, err := g.Unit.Duration()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ret = next.UnixNano() / int64(unitDuration)
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
type UniformFloatGenerator struct {
|
||||
R *rand.Rand
|
||||
MinFloat float64
|
||||
MaxFloat float64
|
||||
}
|
||||
|
||||
func (g *UniformFloatGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
return g.R.Float64()*(g.MaxFloat-g.MinFloat) + g.MinFloat, nil
|
||||
}
|
||||
|
||||
type FixedProbabilitySourceFieldGenerator struct {
|
||||
r *rand.Rand
|
||||
source []string
|
||||
probabilities []float64
|
||||
}
|
||||
|
||||
func NewFixedProbabilitySourceFieldGenerator(vals []string, probs []float64, r *rand.Rand) (*FixedProbabilitySourceFieldGenerator, error) {
|
||||
total := float64(0)
|
||||
numUnspecified := 0
|
||||
for _, prob := range probs {
|
||||
if prob < 0 {
|
||||
numUnspecified++
|
||||
continue
|
||||
}
|
||||
total += prob
|
||||
}
|
||||
if total > 1.0 {
|
||||
return nil, errors.Errorf("sum of all probabilities is greater than 1: %f", total)
|
||||
}
|
||||
unspecProb := 0.0
|
||||
if numUnspecified > 0 {
|
||||
unspecProb = (1.0 - total) / float64(numUnspecified)
|
||||
}
|
||||
total = float64(0)
|
||||
for i, prob := range probs {
|
||||
if prob < 0 {
|
||||
probs[i] = unspecProb
|
||||
}
|
||||
total += prob
|
||||
probs[i] = total
|
||||
}
|
||||
return &FixedProbabilitySourceFieldGenerator{
|
||||
r: r,
|
||||
source: vals,
|
||||
probabilities: probs,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *FixedProbabilitySourceFieldGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
val := g.r.Float64()
|
||||
idx := sort.SearchFloat64s(g.probabilities, val)
|
||||
if idx == len(g.probabilities) {
|
||||
return nil, nil
|
||||
}
|
||||
return g.source[idx], nil
|
||||
}
|
||||
|
||||
type ShiftingStringGenerator struct {
|
||||
shiftAmnt uint64
|
||||
z *rand.Zipf
|
||||
i uint64
|
||||
step uint64
|
||||
baseLen uint64
|
||||
lenVariance uint64
|
||||
}
|
||||
|
||||
func (g *ShiftingStringGenerator) Generate(_ interface{}) (interface{}, error) {
|
||||
num := g.z.Uint64() + g.shiftAmnt
|
||||
g.i++
|
||||
if g.i%g.step == 0 {
|
||||
g.shiftAmnt++
|
||||
}
|
||||
length := g.baseLen + num%(g.lenVariance+1)
|
||||
return numToString(num, length), nil
|
||||
}
|
||||
|
||||
func numToString(num uint64, length uint64) string {
|
||||
return fmt.Sprintf("%0*X", length, num)
|
||||
}
|
||||
|
||||
func NewShiftingStringGenerator(g *GenField, r *rand.Rand) (*ShiftingStringGenerator, error) {
|
||||
if g.MinLen+g.MaxLen != 0 {
|
||||
if g.MinLen == 0 {
|
||||
g.MinLen = g.MaxLen
|
||||
} else if g.MaxLen == 0 {
|
||||
g.MaxLen = g.MinLen
|
||||
}
|
||||
}
|
||||
if g.MaxLen < g.MinLen {
|
||||
return nil, errors.Errorf("max_len(%d) must be greater than or equal to min_len(%d)", g.MaxLen, g.MinLen)
|
||||
}
|
||||
if g.Step < 0 {
|
||||
return nil, errors.Errorf("'step' for shifting must be positive, but got: %d", g.Step)
|
||||
}
|
||||
if g.Step == 0 {
|
||||
g.Step = 1 // default to 1 if no step is passed
|
||||
}
|
||||
if g.S == 0 {
|
||||
g.S = 1.01
|
||||
}
|
||||
if g.V == 0 {
|
||||
g.V = 200
|
||||
}
|
||||
z := rand.NewZipf(r, g.S, g.V, g.Cardinality-1)
|
||||
return &ShiftingStringGenerator{
|
||||
z: z,
|
||||
step: uint64(g.Step),
|
||||
baseLen: g.MinLen,
|
||||
lenVariance: g.MaxLen - g.MinLen,
|
||||
}, nil
|
||||
}
|
||||
309
idk/datagen/custom_test.go
Normal file
309
idk/datagen/custom_test.go
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
)
|
||||
|
||||
func TestGetIDKFields(t *testing.T) {
|
||||
cc := &CustomConfig{
|
||||
Fields: []*GenField{
|
||||
{
|
||||
Name: "a",
|
||||
Type: "uint",
|
||||
},
|
||||
{
|
||||
Name: "b",
|
||||
Type: "int",
|
||||
},
|
||||
},
|
||||
IDKParams: IDKParams{
|
||||
Fields: map[string][]IngestField{
|
||||
"a": {
|
||||
{
|
||||
Type: "ID",
|
||||
Name: "a",
|
||||
},
|
||||
{
|
||||
Type: "Int",
|
||||
Name: "a2",
|
||||
},
|
||||
{
|
||||
Type: "ID",
|
||||
Name: "ID_ttl",
|
||||
TimeQuantum: "YMD",
|
||||
TTL: "101s",
|
||||
},
|
||||
{
|
||||
Type: "IDArray",
|
||||
Name: "IDArray_ttl",
|
||||
TimeQuantum: "YMD",
|
||||
TTL: "102s",
|
||||
},
|
||||
{
|
||||
Type: "String",
|
||||
Name: "String_ttl",
|
||||
TimeQuantum: "YMD",
|
||||
TTL: "103s",
|
||||
},
|
||||
{
|
||||
Type: "StringArray",
|
||||
Name: "StringArray_ttl",
|
||||
TimeQuantum: "YMD",
|
||||
TTL: "104s",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
min := int64(0)
|
||||
max := int64(0)
|
||||
exp := &IDKAndGenFields{
|
||||
schema: []idk.Field{
|
||||
idk.IDField{
|
||||
NameVal: "a",
|
||||
DestNameVal: "a",
|
||||
},
|
||||
idk.IntField{
|
||||
NameVal: "a",
|
||||
DestNameVal: "a2",
|
||||
},
|
||||
idk.IDField{
|
||||
NameVal: "a",
|
||||
DestNameVal: "ID_ttl",
|
||||
Quantum: "YMD",
|
||||
TTL: "101s",
|
||||
},
|
||||
idk.IDArrayField{
|
||||
NameVal: "a",
|
||||
DestNameVal: "IDArray_ttl",
|
||||
Quantum: "YMD",
|
||||
TTL: "102s",
|
||||
},
|
||||
idk.StringField{
|
||||
NameVal: "a",
|
||||
DestNameVal: "String_ttl",
|
||||
Quantum: "YMD",
|
||||
TTL: "103s",
|
||||
},
|
||||
idk.StringArrayField{
|
||||
NameVal: "a",
|
||||
DestNameVal: "StringArray_ttl",
|
||||
Quantum: "YMD",
|
||||
TTL: "104s",
|
||||
},
|
||||
idk.IntField{
|
||||
NameVal: "b",
|
||||
DestNameVal: "", // no dest because we take the default path
|
||||
Min: &min,
|
||||
Max: &max,
|
||||
},
|
||||
},
|
||||
genFields: []*GenField{
|
||||
{
|
||||
Name: "a",
|
||||
Type: "uint",
|
||||
},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
{
|
||||
Name: "b",
|
||||
Type: "int",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
igs, err := cc.GetIDKFields()
|
||||
if err != nil {
|
||||
t.Fatalf("getting IDK fields: %v", err)
|
||||
}
|
||||
|
||||
if len(igs.schema) != len(exp.schema) ||
|
||||
len(igs.genFields) != len(exp.genFields) ||
|
||||
len(igs.schema) != len(igs.genFields) {
|
||||
t.Fatalf("mismatched lengths: %+v", igs)
|
||||
}
|
||||
for i := 0; i < len(igs.schema); i++ {
|
||||
gotSchema, expSchema := igs.schema[i], exp.schema[i]
|
||||
gotGenField, expGenField := igs.genFields[i], exp.genFields[i]
|
||||
if !idk.FieldsEqual(gotSchema, expSchema) {
|
||||
t.Fatalf("mismatched schemas at %d: got:\n%+v\nexp:\n%+v", i, gotSchema, expSchema)
|
||||
}
|
||||
if gotGenField == nil && expGenField == nil {
|
||||
// great
|
||||
} else if *gotGenField != *expGenField {
|
||||
t.Fatalf("mismatched genFields at %d: got:\n%+v\nexp:\n%+v", i, gotGenField, expGenField)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestNewCustomUnmarshalling(t *testing.T) {
|
||||
tmp, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
t.Fatalf("creating temp file: %v", err)
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
_, err = tmp.WriteString(`
|
||||
fields:
|
||||
- name: "myfield"
|
||||
type: "timestamp"
|
||||
min_step_duration: "1s"
|
||||
min_date: 2013-10-16T22:34:43Z
|
||||
time_format: "unix"
|
||||
time_unit: "ms"
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("writing: %v", err)
|
||||
}
|
||||
cc := NewCustom(SourceGeneratorConfig{CustomConfig: tmp.Name()}).(*Custom)
|
||||
if cc.err != nil {
|
||||
t.Fatalf("creating Custom: %v", cc.err)
|
||||
}
|
||||
if cc.CustomConfig.Fields[0].minStepDuration != time.Second {
|
||||
t.Fatalf("wrong duration: %+v", cc.CustomConfig.Fields[0])
|
||||
}
|
||||
if cc.CustomConfig.Fields[0].MinDate.Hour() != 22 {
|
||||
t.Fatalf("wrong time: %s", cc.CustomConfig.Fields[0].MinDate)
|
||||
}
|
||||
if cc.CustomConfig.Fields[0].TimeUnit != idk.Millisecond {
|
||||
t.Fatalf("wrong time unit: %s", cc.CustomConfig.Fields[0].TimeUnit)
|
||||
}
|
||||
if cc.CustomConfig.Fields[0].TimeFormat != unixFormat {
|
||||
t.Fatalf("wrong time format: %s", cc.CustomConfig.Fields[0].TimeFormat)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestIncreasingTimestampGenerator(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
unit idk.Unit
|
||||
min int64
|
||||
max int64
|
||||
}{
|
||||
{
|
||||
name: "millis",
|
||||
unit: idk.Millisecond,
|
||||
max: 1136073600000 + 2001,
|
||||
min: 1136073600000,
|
||||
},
|
||||
{
|
||||
name: "days",
|
||||
unit: idk.Day,
|
||||
max: 13149,
|
||||
min: 13149,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testcase := range cases {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
genField := &GenField{
|
||||
Type: "timestamp",
|
||||
minStepDuration: time.Second,
|
||||
maxStepDuration: time.Second * 2,
|
||||
TimeFormat: unixFormat,
|
||||
TimeUnit: testcase.unit,
|
||||
Distribution: "increasing",
|
||||
MinDate: time.Date(2006, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
MaxDate: time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
r := rand.New(rand.NewSource(3))
|
||||
gen, err := genField.Generator(r)
|
||||
if err != nil {
|
||||
t.Fatalf("making generator: %v", err)
|
||||
}
|
||||
|
||||
rec, err := gen.Generate(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("generating: %v", err)
|
||||
}
|
||||
if recInt, ok := rec.(int64); !ok {
|
||||
t.Fatalf("expected int64, but got %v of %[1]T", recInt)
|
||||
} else if recInt < testcase.min || recInt > testcase.max {
|
||||
t.Fatalf("unexpected value for time: %v", recInt)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShiftingStringGenerator(t *testing.T) {
|
||||
// max < min
|
||||
if ssg, err := NewShiftingStringGenerator(&GenField{MinLen: 10, MaxLen: 9}, nil); err == nil {
|
||||
t.Fatalf("expected error, but got %+v", ssg)
|
||||
} else if !strings.Contains(err.Error(), "must be greater than or equal to") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
// neg step
|
||||
if ssg, err := NewShiftingStringGenerator(&GenField{Step: -1}, nil); err == nil {
|
||||
t.Fatalf("expected error, but got %+v", ssg)
|
||||
} else if !strings.Contains(err.Error(), "'step' for shifting must be positive") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
// min zero but max not
|
||||
if ssg, err := NewShiftingStringGenerator(&GenField{MaxLen: 7, Step: 2}, rand.New(rand.NewSource(1))); err != nil {
|
||||
t.Fatalf("unexpected error %v", err)
|
||||
} else if ssg.lenVariance != 0 {
|
||||
t.Fatalf("expected zero variance, but got: %+v", ssg)
|
||||
}
|
||||
|
||||
// max zero but min not
|
||||
if ssg, err := NewShiftingStringGenerator(&GenField{MinLen: 7}, rand.New(rand.NewSource(1))); err != nil {
|
||||
t.Fatalf("unexpected error %v", err)
|
||||
} else if ssg.lenVariance != 0 {
|
||||
t.Fatalf("expected zero variance, but got: %+v", ssg)
|
||||
} else if ssg.step != 1 {
|
||||
t.Fatalf("expected step 0 but got: %+v", ssg)
|
||||
}
|
||||
|
||||
ssg, err := NewShiftingStringGenerator(&GenField{MinLen: 5, MaxLen: 10, Cardinality: 4, Step: 20}, rand.New(rand.NewSource(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %+v", err)
|
||||
}
|
||||
|
||||
vals := make([]string, 0, 40)
|
||||
valMap := make(map[string]struct{})
|
||||
for i := 0; i < 20; i++ {
|
||||
val, err := ssg.Generate(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected generation err: %v", err)
|
||||
}
|
||||
valStr := val.(string)
|
||||
vals = append(vals, valStr)
|
||||
valMap[valStr] = struct{}{}
|
||||
if len(valStr) < 5 || len(valStr) > 10 {
|
||||
t.Fatalf("got unexpected length val at position: %d, %v", i, vals)
|
||||
}
|
||||
}
|
||||
if len(valMap) != 4 {
|
||||
t.Fatalf("unexpected cardinality: %v", valMap)
|
||||
}
|
||||
for i := 20; i < 40; i++ {
|
||||
val, err := ssg.Generate(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected generation err: %v", err)
|
||||
}
|
||||
valStr := val.(string)
|
||||
vals = append(vals, valStr)
|
||||
delete(valMap, valStr)
|
||||
if len(valStr) < 5 || len(valStr) > 10 {
|
||||
t.Fatalf("got unexpected length val at position: %d, %v", i, vals)
|
||||
}
|
||||
}
|
||||
if len(valMap) != 1 {
|
||||
t.Fatalf("should have a single value from the first 20 that isn't generated in second 20, but have valMap:\n%v\nvals:\n%v", valMap, vals)
|
||||
}
|
||||
}
|
||||
106
idk/datagen/customer.go
Normal file
106
idk/datagen/customer.go
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Customer implements interface.
|
||||
var _ Sourcer = (*Customer)(nil)
|
||||
|
||||
// Customer implements Sourcer.
|
||||
type Customer struct{}
|
||||
|
||||
// NewCustomer returns a new instance of Customer.
|
||||
func NewCustomer(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Customer{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (c *Customer) Source(cfg SourceConfig) idk.Source {
|
||||
src := &CustomerSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"},
|
||||
idk.StringField{NameVal: "region"},
|
||||
idk.IntField{NameVal: "size"},
|
||||
idk.IDField{NameVal: "sales_channel"},
|
||||
},
|
||||
}
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
|
||||
src.regionZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(regions))-1)
|
||||
|
||||
src.record[0] = cfg.startFrom
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (c *Customer) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (c *Customer) DefaultEndAt() uint64 {
|
||||
return 100 // TODO: what should this be?
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (c *Customer) Info() string {
|
||||
return "TODO"
|
||||
}
|
||||
|
||||
// Ensure CustomerSource implements interface.
|
||||
var _ idk.Source = (*CustomerSource)(nil)
|
||||
|
||||
type CustomerSource struct {
|
||||
Log logger.Logger
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
rand *rand.Rand
|
||||
regionZipf *rand.Zipf
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
var regions = []string{"US", "Canada", "Central America", "South America", "Western Europe", "Eastern Europe", "Africa", "China", "Japan", "Russia", "Middle East", "Pacific Islands", "Antarctica"}
|
||||
|
||||
func (s *CustomerSource) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.cur
|
||||
s.record[1] = regions[s.regionZipf.Uint64()]
|
||||
s.record[2] = s.rand.Intn(24990) + 10
|
||||
s.record[3] = uint64(s.rand.Intn(500))
|
||||
s.cur++
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *CustomerSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *CustomerSource) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*CustomerSource)(nil)
|
||||
|
||||
func (s *CustomerSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
603
idk/datagen/customer_segmentation.go
Normal file
603
idk/datagen/customer_segmentation.go
Normal file
|
|
@ -0,0 +1,603 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
)
|
||||
|
||||
/*
|
||||
go run cmd/datagen/main.go \
|
||||
--source=customer_segmentation --pilosa.index=customer_segmentation \
|
||||
--start-from=1 --end-at=100 --pilosa.batch-size=100
|
||||
|
||||
go run cmd/datagen/main.go \
|
||||
--source=example --pilosa.index=example \
|
||||
--start-from=1 --end-at=100 --pilosa.batch-size=100
|
||||
*/
|
||||
|
||||
// Ensure Person implements interface.
|
||||
var _ Sourcer = (*Person)(nil)
|
||||
|
||||
// Person implements Sourcer,
|
||||
type Person struct{}
|
||||
|
||||
// NewPerson returns a new instance of Person.
|
||||
func NewPerson(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Person{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (p *Person) Source(cfg SourceConfig) idk.Source {
|
||||
g := gen.New(gen.OptGenSeed(cfg.seed))
|
||||
src := &PersonSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "sex"}, // 1
|
||||
idk.IntField{NameVal: "age"}, // 2
|
||||
idk.StringField{NameVal: "race"}, // 3
|
||||
idk.BoolField{NameVal: "hispanic"}, // 4
|
||||
idk.StringField{NameVal: "political_party_affiliation"}, // 5
|
||||
idk.StringArrayField{NameVal: "education"}, // 6
|
||||
idk.IntField{NameVal: "income"}, // 7
|
||||
idk.IDField{NameVal: "general_election_voting_frequency"}, // 8
|
||||
idk.IDField{NameVal: "primary_election_voting_frequency"}, // 9
|
||||
idk.StringArrayField{NameVal: "hobbies"}, // 10
|
||||
},
|
||||
g: g,
|
||||
generatePerson: getGenRandPerson(g.R),
|
||||
hobbyStash: []string{},
|
||||
}
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[0] = uint64(0)
|
||||
src.record[1] = make([]byte, 12)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (e *Person) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (e *Person) DefaultEndAt() uint64 {
|
||||
return 100
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (p *Person) Info() string {
|
||||
return "Generates data for customer segmentation."
|
||||
}
|
||||
|
||||
// Ensure PersonSource implements interface.
|
||||
var _ idk.Source = (*PersonSource)(nil)
|
||||
|
||||
// PersonSource is an instance of a source generated
|
||||
// by the Sourcer implementation Person.
|
||||
type PersonSource struct {
|
||||
cur uint64
|
||||
endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
|
||||
g *gen.Gen
|
||||
generatePerson func() *person
|
||||
hobbyStash []string
|
||||
}
|
||||
|
||||
// Record implements idk.Source.
|
||||
func (ps *PersonSource) Record() (idk.Record, error) {
|
||||
if ps.cur >= ps.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
p := ps.generatePerson()
|
||||
ps.setRandomHobbies()
|
||||
|
||||
// Populate fields
|
||||
ps.record[0] = ps.cur
|
||||
ps.record[1] = p.sex
|
||||
ps.record[2] = p.age
|
||||
ps.record[3] = p.race
|
||||
ps.record[4] = p.hispanic
|
||||
ps.record[5] = p.politicalPartyAffiliation
|
||||
ps.record[6] = educationLevelToStringArr(p.education)
|
||||
ps.record[7] = p.income
|
||||
ps.record[8] = p.generalElectionVotingFrequency
|
||||
ps.record[9] = p.primaryElectionVotingFrequency
|
||||
ps.record[10] = ps.hobbyStash
|
||||
|
||||
// Increment the ID.
|
||||
ps.cur++
|
||||
|
||||
return ps.record, nil
|
||||
}
|
||||
|
||||
func (ps *PersonSource) setRandomHobbies() {
|
||||
ps.hobbyStash = ps.g.StringSliceFromListWeighted(ps.hobbyStash, hobbies, 2, 5)
|
||||
}
|
||||
|
||||
// Schema implements idk.Source.
|
||||
func (ps *PersonSource) Schema() []idk.Field {
|
||||
return ps.schema
|
||||
}
|
||||
|
||||
// ----------------------------------------------------
|
||||
// THIS IS WHERE THE MAGIC HAPPENS
|
||||
// ----------------------------------------------------
|
||||
|
||||
type randValue struct {
|
||||
value interface{}
|
||||
probability float64 // probability of being picked, all in set should add to 1.0
|
||||
count int // number of times val has been picked so far
|
||||
}
|
||||
|
||||
type randVarPickerFn func() interface{}
|
||||
type getStatsFn func() string
|
||||
|
||||
func getRandVarPicker(rng *rand.Rand, vals []randValue) (randVarPickerFn, getStatsFn) {
|
||||
|
||||
cumulativeSums := make([]float64, len(vals))
|
||||
currSum := float64(0)
|
||||
for i, v := range vals {
|
||||
currSum += v.probability
|
||||
cumulativeSums[i] = currSum
|
||||
}
|
||||
|
||||
// call stats at any time to get, well, stats
|
||||
stats := func() string {
|
||||
// make copy to avoid modifying original
|
||||
valsCopy := make([]randValue, len(vals))
|
||||
copy(valsCopy, vals)
|
||||
// sort counts
|
||||
sort.Slice(valsCopy, func(i, j int) bool {
|
||||
return valsCopy[i].count > valsCopy[j].count
|
||||
})
|
||||
// get total gen so far
|
||||
total := 0
|
||||
for _, v := range valsCopy {
|
||||
total += v.count
|
||||
}
|
||||
// gen str summary
|
||||
var sb strings.Builder
|
||||
for _, v := range valsCopy {
|
||||
percentage := (float64(v.count) / float64(total)) * 100
|
||||
s := fmt.Sprintf("%v (%d/%d) %0.2f%%\n", v.value, v.count, total, percentage)
|
||||
sb.WriteString(s)
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
picker := func() interface{} {
|
||||
rv := rng.Float64()
|
||||
i := sort.SearchFloat64s(cumulativeSums, rv)
|
||||
// note that if the probabilities do not add up to 1.0
|
||||
// i will == len(cumulativeSums), hence out of bounds error
|
||||
// however, floating points are tricky and require special care
|
||||
// so I will assume that the probabilities provided were meant to
|
||||
// add closely up to 1.0 (eg it ended up being 0.999900)
|
||||
// and if the rv generated is for example 0.9999000011, the last
|
||||
// value of the slice will be picked instead. In future, there
|
||||
// should be an assertion on the vals slice probabilities to ensure
|
||||
// they add up to 1 +/- some epsilon value.
|
||||
if i == len(vals) {
|
||||
i = len(vals) - 1
|
||||
}
|
||||
vals[i].count++
|
||||
return vals[i].value
|
||||
}
|
||||
|
||||
return picker, stats
|
||||
}
|
||||
|
||||
type numericRange struct {
|
||||
min int // inclusive
|
||||
max int // inclusive
|
||||
}
|
||||
|
||||
type raceEthnicity struct {
|
||||
race string
|
||||
isHispanic bool
|
||||
}
|
||||
|
||||
type educationLevel struct {
|
||||
hasCompletedHighSchool bool
|
||||
hasAttendedSomeCollege bool
|
||||
hasCompletedBachelors bool
|
||||
hasCompletedGraduate bool
|
||||
}
|
||||
|
||||
var cG []string = []string{"High school diploma or GED", "Some college", "Bachelor's degree", "Doctorate and/or Professional degree"}
|
||||
var cB []string = []string{"High school diploma or GED", "Some college", "Bachelor's degree"}
|
||||
var cASG []string = []string{"High school diploma or GED", "Some college"}
|
||||
var cHS []string = []string{"High school diploma or GED"}
|
||||
|
||||
func educationLevelToStringArr(e educationLevel) []string {
|
||||
if e.hasCompletedGraduate {
|
||||
return cG
|
||||
}
|
||||
if e.hasCompletedBachelors {
|
||||
return cB
|
||||
}
|
||||
if e.hasAttendedSomeCollege {
|
||||
return cASG
|
||||
}
|
||||
if e.hasCompletedHighSchool {
|
||||
return cHS
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pickRandNumFromRange(rng *rand.Rand, nr numericRange) int {
|
||||
upperBoundExclusive := int32((nr.max - nr.min) + 1)
|
||||
rv := int(rng.Int31n(upperBoundExclusive)) // [0, upperBoundExclusive)
|
||||
return rv + nr.min
|
||||
}
|
||||
|
||||
type person struct {
|
||||
sex string
|
||||
age int
|
||||
race string
|
||||
hispanic bool
|
||||
politicalPartyAffiliation string
|
||||
education educationLevel
|
||||
income int
|
||||
generalElectionVotingFrequency int
|
||||
primaryElectionVotingFrequency int
|
||||
}
|
||||
|
||||
func getGenRandPerson(rng *rand.Rand) func() *person {
|
||||
|
||||
pickSex, _ := getRandVarPicker(rng, []randValue{
|
||||
{
|
||||
value: "Male",
|
||||
probability: 0.475,
|
||||
},
|
||||
{
|
||||
value: "Female",
|
||||
probability: 0.475,
|
||||
},
|
||||
{
|
||||
value: "Unspecified",
|
||||
probability: 0.05,
|
||||
},
|
||||
})
|
||||
|
||||
pickAge, _ := getRandVarPicker(rng, []randValue{
|
||||
{
|
||||
value: numericRange{
|
||||
min: 0,
|
||||
max: 14,
|
||||
},
|
||||
probability: 0.1873,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 15,
|
||||
max: 24,
|
||||
},
|
||||
probability: 0.1327,
|
||||
}, {
|
||||
value: numericRange{
|
||||
min: 25,
|
||||
max: 54,
|
||||
},
|
||||
probability: 0.3945,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 55,
|
||||
max: 64,
|
||||
},
|
||||
probability: 0.1291,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 65,
|
||||
max: 100,
|
||||
},
|
||||
probability: 0.1563,
|
||||
},
|
||||
})
|
||||
|
||||
// 75.72% (53.89% Non-Hispanic) White
|
||||
// 16.06% (14.77% Non-Hispanic) African American
|
||||
// 7.09% Asian / Pacific Islander
|
||||
// 1.13% Native American / Alaskan Native
|
||||
pickRaceAndEthnicity, _ := getRandVarPicker(rng, []randValue{
|
||||
{
|
||||
value: raceEthnicity{
|
||||
race: "White",
|
||||
isHispanic: false,
|
||||
},
|
||||
probability: 0.5389,
|
||||
},
|
||||
{
|
||||
value: raceEthnicity{
|
||||
race: "White",
|
||||
isHispanic: true,
|
||||
},
|
||||
probability: 0.2183,
|
||||
},
|
||||
{
|
||||
value: raceEthnicity{
|
||||
race: "African American",
|
||||
isHispanic: false,
|
||||
},
|
||||
probability: 0.1477,
|
||||
},
|
||||
{
|
||||
value: raceEthnicity{
|
||||
race: "African American",
|
||||
isHispanic: true,
|
||||
},
|
||||
probability: 0.0129,
|
||||
},
|
||||
{
|
||||
value: raceEthnicity{
|
||||
race: "Asian / Pacific Islander",
|
||||
isHispanic: false,
|
||||
},
|
||||
probability: 0.0709,
|
||||
},
|
||||
{
|
||||
value: raceEthnicity{
|
||||
race: "Native American / Alaskan Native",
|
||||
isHispanic: false,
|
||||
},
|
||||
probability: 0.0113,
|
||||
},
|
||||
})
|
||||
|
||||
// Democrat 27%
|
||||
// Republican 20%
|
||||
// Independent 19%
|
||||
// Democrat & Independent 6%
|
||||
// Republican & Independent 5%
|
||||
// Democrat & Republican 4%
|
||||
// Democrat & Republican & Independent 1%
|
||||
// None Reported/ Declined to reply 18%
|
||||
pickPoliticalPartyAffiliation, _ := getRandVarPicker(rng, []randValue{
|
||||
{
|
||||
value: "Democrat",
|
||||
probability: 0.27,
|
||||
},
|
||||
{
|
||||
value: "Republican",
|
||||
probability: 0.20,
|
||||
},
|
||||
{
|
||||
value: "Independent",
|
||||
probability: 0.19,
|
||||
},
|
||||
{
|
||||
value: "Democrat and Independent",
|
||||
probability: 0.06,
|
||||
},
|
||||
{
|
||||
value: "Republican and Independent",
|
||||
probability: 0.05,
|
||||
},
|
||||
{
|
||||
value: "Democrat and Republican",
|
||||
probability: 0.04,
|
||||
},
|
||||
{
|
||||
value: "Democrat and Republican and Independent",
|
||||
probability: 0.01,
|
||||
},
|
||||
{
|
||||
value: "",
|
||||
probability: 0.18,
|
||||
},
|
||||
})
|
||||
|
||||
// 89.80% High school diploma or GED
|
||||
// 61.28% Some college
|
||||
// 34.98% Bachelor's degree
|
||||
// 13.04% Master's and/or doctorate and/or professional degree
|
||||
|
||||
// Percentages are additive, meaning that 89.80% High School/GED
|
||||
// includes everyone with that education level or higher.
|
||||
|
||||
// Let
|
||||
// H = HighSchool
|
||||
// SC = SomeCollege
|
||||
// B = Bachelor's
|
||||
// G = Graduate/Professional
|
||||
|
||||
// P(H) = 0.8980
|
||||
// P(H & SC) = 0.6128
|
||||
// P(H & SC & B) = 0.3498
|
||||
// P(H & SC & B & G) = 0.1304
|
||||
|
||||
// P(SC | H) = P(H & SC) / P(H) = 0.6824
|
||||
// P(B | H & SC) = P(H & SC & B) / P(H & SC) = 0.5708
|
||||
// P(G | H & SC & B) = P(H & SC & B & G) / P(SC & B & G) = 0.3728
|
||||
pickEducationLevel := func(rng *rand.Rand) educationLevel {
|
||||
var ed educationLevel
|
||||
if rng.Float64() > 0.8980 {
|
||||
return ed
|
||||
}
|
||||
ed.hasCompletedHighSchool = true
|
||||
if rng.Float64() > 0.6824 {
|
||||
return ed
|
||||
}
|
||||
ed.hasAttendedSomeCollege = true
|
||||
if rng.Float64() > 0.5708 {
|
||||
return ed
|
||||
}
|
||||
ed.hasCompletedBachelors = true
|
||||
if rng.Float64() > 0.3728 {
|
||||
return ed
|
||||
}
|
||||
ed.hasCompletedGraduate = true
|
||||
return ed
|
||||
}
|
||||
|
||||
// 9.1% $0-$15000
|
||||
// 8% $15000-$24999
|
||||
// 8.3% $25000-$34999
|
||||
// 11.7% $35000-$49999
|
||||
// 16.5% $50000-$74999
|
||||
// 12.3% 75000-99999
|
||||
// 15.5% 100000-149999
|
||||
// 8.3% 150000-199999
|
||||
// 10.3% $200000+
|
||||
pickIncome, _ := getRandVarPicker(rng, []randValue{
|
||||
{
|
||||
value: numericRange{
|
||||
min: 0,
|
||||
max: 15000,
|
||||
},
|
||||
probability: 0.091,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 15000,
|
||||
max: 24999,
|
||||
},
|
||||
probability: 0.08,
|
||||
}, {
|
||||
value: numericRange{
|
||||
min: 25000,
|
||||
max: 34999,
|
||||
},
|
||||
probability: 0.083,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 35000,
|
||||
max: 49999,
|
||||
},
|
||||
probability: 0.117,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 50000,
|
||||
max: 74999,
|
||||
},
|
||||
probability: 0.165,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 75000,
|
||||
max: 99999,
|
||||
},
|
||||
probability: 0.123,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 100000,
|
||||
max: 149999,
|
||||
},
|
||||
probability: 0.155,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 150000,
|
||||
max: 199999,
|
||||
},
|
||||
probability: 0.083,
|
||||
},
|
||||
{
|
||||
value: numericRange{
|
||||
min: 200000,
|
||||
max: 500000,
|
||||
},
|
||||
probability: 0.103,
|
||||
},
|
||||
})
|
||||
|
||||
// 0 -> 21%
|
||||
// 1 -> 17%
|
||||
// 2 -> 17%
|
||||
// 3 -> 15%
|
||||
// 4 -> 38%
|
||||
pickGeneralElectionVotingFrequency, _ := getRandVarPicker(rng, []randValue{
|
||||
{
|
||||
value: 0,
|
||||
probability: 0.21,
|
||||
},
|
||||
{
|
||||
value: 1,
|
||||
probability: 0.17,
|
||||
},
|
||||
{
|
||||
value: 2,
|
||||
probability: 0.17,
|
||||
},
|
||||
{
|
||||
value: 3,
|
||||
probability: 0.15,
|
||||
},
|
||||
{
|
||||
value: 4,
|
||||
probability: 0.38,
|
||||
},
|
||||
})
|
||||
|
||||
// 0 -> 51%
|
||||
// 1 -> 19%
|
||||
// 2 -> 11%
|
||||
// 3 -> 7%
|
||||
// 4 -> 11%
|
||||
pickPrimaryElectionVotingFrequency, _ := getRandVarPicker(rng, []randValue{
|
||||
{
|
||||
value: 0,
|
||||
probability: 0.51,
|
||||
},
|
||||
{
|
||||
value: 1,
|
||||
probability: 0.19,
|
||||
},
|
||||
{
|
||||
value: 2,
|
||||
probability: 0.11,
|
||||
},
|
||||
{
|
||||
value: 3,
|
||||
probability: 0.07,
|
||||
},
|
||||
{
|
||||
value: 4,
|
||||
probability: 0.11,
|
||||
},
|
||||
})
|
||||
|
||||
return func() *person {
|
||||
p := &person{}
|
||||
p.sex = pickSex().(string)
|
||||
p.age = pickRandNumFromRange(rng, pickAge().(numericRange))
|
||||
raceAndEthnicity := pickRaceAndEthnicity().(raceEthnicity)
|
||||
p.race = raceAndEthnicity.race
|
||||
p.hispanic = raceAndEthnicity.isHispanic
|
||||
p.politicalPartyAffiliation = pickPoliticalPartyAffiliation().(string)
|
||||
p.education = pickEducationLevel(rng)
|
||||
p.income = pickRandNumFromRange(rng, pickIncome().(numericRange))
|
||||
p.generalElectionVotingFrequency = pickGeneralElectionVotingFrequency().(int)
|
||||
p.primaryElectionVotingFrequency = pickPrimaryElectionVotingFrequency().(int)
|
||||
return p
|
||||
}
|
||||
}
|
||||
|
||||
func (ps *PersonSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
95
idk/datagen/customer_segmentation_add_linkedin.go
Normal file
95
idk/datagen/customer_segmentation_add_linkedin.go
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
)
|
||||
|
||||
var _ Sourcer = (*LinkedInPerson)(nil)
|
||||
|
||||
type LinkedInPerson struct{}
|
||||
|
||||
func NewLinkedInPerson(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &LinkedInPerson{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (p *LinkedInPerson) Source(cfg SourceConfig) idk.Source {
|
||||
g := gen.New(gen.OptGenSeed(cfg.seed))
|
||||
src := &LinkedInPersonSource{
|
||||
startFrom: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "city"}, // 1
|
||||
idk.StringField{NameVal: "zip_code"}, // 2
|
||||
idk.StringArrayField{NameVal: "skills"}, // 3
|
||||
idk.StringArrayField{NameVal: "titles"}, // 4
|
||||
},
|
||||
g: g,
|
||||
}
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[3] = []string{}
|
||||
src.record[4] = []string{}
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (e *LinkedInPerson) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (e *LinkedInPerson) DefaultEndAt() uint64 {
|
||||
return 100
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (p *LinkedInPerson) Info() string {
|
||||
return "Simulates adding LinkedIn data to existing customer segmentation data."
|
||||
}
|
||||
|
||||
var _ idk.Source = (*LinkedInPersonSource)(nil)
|
||||
|
||||
type LinkedInPersonSource struct {
|
||||
count uint64
|
||||
startFrom uint64
|
||||
endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
|
||||
g *gen.Gen
|
||||
}
|
||||
|
||||
func (lp *LinkedInPersonSource) Record() (idk.Record, error) {
|
||||
lp.count++
|
||||
if lp.count > lp.endAt-lp.startFrom {
|
||||
return nil, io.EOF
|
||||
}
|
||||
lp.record[0] = uint64(lp.g.R.Int63n(int64(lp.endAt-lp.startFrom))) + lp.startFrom
|
||||
lp.record[1] = lp.g.StringFromList(uscities)
|
||||
lp.record[2] = lp.g.StringFromList(zip_codes)
|
||||
lp.record[3] = lp.g.StringSliceFromListWeighted(lp.record[3].([]string), skills, 2, 5)
|
||||
lp.record[4] = lp.g.StringSliceFromListWeighted(lp.record[4].([]string), titles, 2, 5)
|
||||
|
||||
return lp.record, nil
|
||||
}
|
||||
|
||||
func (lp *LinkedInPersonSource) Schema() []idk.Field {
|
||||
return lp.schema
|
||||
}
|
||||
func (lp *LinkedInPersonSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
27
idk/datagen/customer_segmentation_test.go
Normal file
27
idk/datagen/customer_segmentation_test.go
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPersonHobbies(t *testing.T) {
|
||||
p := NewPerson(SourceGeneratorConfig{})
|
||||
ps := p.Source(SourceConfig{endAt: 10})
|
||||
|
||||
i := 0
|
||||
for i = 0; i < 10; i++ {
|
||||
rec, err := ps.Record()
|
||||
if err != nil {
|
||||
t.Fatalf("getting record: %v", err)
|
||||
}
|
||||
data := rec.Data()
|
||||
if dataStringSlice, ok := data[10].([]string); !ok {
|
||||
t.Fatalf("expected string slice, but got type:%T, data:%[1]v", data[10])
|
||||
} else if len(dataStringSlice) > 0 {
|
||||
break // we just want to see one that has data in it, then we'll call that success
|
||||
}
|
||||
}
|
||||
if i == 10 {
|
||||
t.Fatalf("never saw any generated hobbies (slice was always empty)")
|
||||
}
|
||||
}
|
||||
559
idk/datagen/datagen_test.go
Normal file
559
idk/datagen/datagen_test.go
Normal file
|
|
@ -0,0 +1,559 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/confluentinc/confluent-kafka-go/kafka"
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/common"
|
||||
)
|
||||
|
||||
var sources = []string{
|
||||
"bank",
|
||||
"claim",
|
||||
"custom",
|
||||
"customer",
|
||||
"dell",
|
||||
"dwarranty",
|
||||
"equipment",
|
||||
"hughes",
|
||||
"item",
|
||||
"kitchensink",
|
||||
"kitchensink_keyed",
|
||||
"merck",
|
||||
"network_ts",
|
||||
"power_scenario_1",
|
||||
"power_scenario_1_2",
|
||||
"power_ts",
|
||||
"site",
|
||||
"sizing",
|
||||
"stringpk",
|
||||
"texas_health",
|
||||
"transactions_scenario_1",
|
||||
"transactions_ts",
|
||||
"warranty",
|
||||
}
|
||||
|
||||
var keyedIndices = map[string]bool{
|
||||
"stringpk": true,
|
||||
"kitchensink_keyed": true,
|
||||
"bank": true,
|
||||
}
|
||||
|
||||
var duplicateIndices = map[string]bool{
|
||||
"bank": true,
|
||||
"customer_segmentation_linkedin": true,
|
||||
"custom": true,
|
||||
}
|
||||
|
||||
// TestConfig tests that all default configurations load.
|
||||
func TestConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, src := range sources {
|
||||
src := src
|
||||
t.Run(src, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
main := NewMain()
|
||||
main.Source = src
|
||||
main.Concurrency = 2
|
||||
if src == "custom" {
|
||||
main.CustomConfig = "testdata/custom.yaml"
|
||||
}
|
||||
main.Target = TargetFeaturebase
|
||||
err := main.Preload()
|
||||
if err != nil {
|
||||
t.Errorf("failed to preload: %v", err)
|
||||
return
|
||||
}
|
||||
main.NoStats()
|
||||
main.PrintPlan()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGen tests record generation.
|
||||
// It ensures that all generated IDs are unique, and that the right number of records are generated.
|
||||
// If the IDs are numerical, it also ensures that they occupy a single contiguous range.
|
||||
func TestGen(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, src := range sources {
|
||||
src := src
|
||||
t.Run(src, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const conc = 16
|
||||
const n = 100
|
||||
|
||||
main := NewMain()
|
||||
main.Source = src
|
||||
main.Concurrency = conc
|
||||
main.EndAt = n - 1
|
||||
if src == "custom" {
|
||||
main.CustomConfig = "testdata/custom.yaml"
|
||||
}
|
||||
if err := main.Preload(); err != nil {
|
||||
t.Errorf("failed to preload: %v", err)
|
||||
return
|
||||
}
|
||||
main.NoStats()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var out [conc]map[interface{}]struct{}
|
||||
var errs [conc]error
|
||||
srcs := main.Sources()
|
||||
for i := range out {
|
||||
i := i
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
ids := map[interface{}]struct{}{}
|
||||
out[i] = ids
|
||||
|
||||
for {
|
||||
rec, err := srcs[i].Record()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return
|
||||
}
|
||||
|
||||
errs[i] = err
|
||||
return
|
||||
}
|
||||
|
||||
id := rec.Data()[0]
|
||||
if idbytes, ok := id.([]byte); ok {
|
||||
id = string(idbytes)
|
||||
}
|
||||
if _, ok := ids[id]; ok {
|
||||
errs[i] = fmt.Errorf("duplicated ID: %v", id)
|
||||
return
|
||||
}
|
||||
ids[id] = struct{}{}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for i, v := range errs {
|
||||
if v != nil {
|
||||
t.Errorf("source %d failed: %v", i, v)
|
||||
}
|
||||
}
|
||||
|
||||
total := 0
|
||||
ids := map[interface{}][]int{}
|
||||
var idlist []interface{}
|
||||
for i, s := range out {
|
||||
total += len(s)
|
||||
for id := range s {
|
||||
if _, ok := ids[id]; !ok {
|
||||
idlist = append(idlist, id)
|
||||
}
|
||||
ids[id] = append(ids[id], i)
|
||||
}
|
||||
}
|
||||
// some sources are expected to have duplicate records, so skip checking for duplicates.
|
||||
if !duplicateIndices[src] {
|
||||
sort.Slice(idlist, func(i, j int) bool {
|
||||
switch idlist[i].(type) {
|
||||
case uint64:
|
||||
return idlist[i].(uint64) < idlist[j].(uint64)
|
||||
case string:
|
||||
return idlist[i].(string) < idlist[j].(string)
|
||||
default:
|
||||
panic(fmt.Errorf("unexpected type: %T", idlist[i]))
|
||||
}
|
||||
})
|
||||
for _, id := range idlist {
|
||||
srcs := ids[id]
|
||||
if len(srcs) > 1 {
|
||||
sort.Ints(srcs)
|
||||
t.Errorf("duplicates of id %v found in sources: %v", id, srcs)
|
||||
}
|
||||
}
|
||||
if base, ok := idlist[0].(uint64); ok {
|
||||
for i, v := range idlist {
|
||||
if base+uint64(i) != v {
|
||||
t.Errorf("found gap in ID range at %d", base+uint64(i))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if total != n {
|
||||
t.Errorf("expected %d records; got %d records", n, total)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func configureTestFlags(main *Main) {
|
||||
if pilosaHost, ok := os.LookupEnv("IDK_TEST_PILOSA_HOST"); ok {
|
||||
main.Pilosa.Hosts = []string{pilosaHost}
|
||||
} else {
|
||||
main.Pilosa.Hosts = []string{"pilosa:10101"}
|
||||
}
|
||||
}
|
||||
|
||||
func TestThousandFormating(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
input uint64
|
||||
output string
|
||||
}{
|
||||
{input: 0, output: "0"},
|
||||
{input: 1000, output: "1,000"},
|
||||
{input: 111222333444, output: "111,222,333,444"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
tc := tc
|
||||
if tc.output != AddThousandSep(tc.input) {
|
||||
t.Errorf("expected %v but got %v", tc.output, AddThousandSep(tc.input))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestIntegration is an intergration test that ingests each source into Pilosa.
|
||||
// It currently only verifies that the correct number of rows are ingested into each index.
|
||||
func TestIntegration(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration tests in short mode")
|
||||
return
|
||||
}
|
||||
|
||||
for _, src := range sources {
|
||||
src := src
|
||||
t.Run(src, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const n = 100
|
||||
|
||||
main := NewMain()
|
||||
configureTestFlags(main)
|
||||
main.Pilosa.Index = fmt.Sprintf("test-%s-%d", src, rand.New(rand.NewSource(time.Now().Unix())).Uint64())
|
||||
main.Source = src
|
||||
main.Target = TargetFeaturebase
|
||||
main.EndAt = n - 1
|
||||
main.Concurrency = 3
|
||||
main.Pilosa.BatchSize = 13
|
||||
if src == "custom" {
|
||||
main.CustomConfig = "testdata/custom.yaml"
|
||||
}
|
||||
if err := main.Preload(); err != nil {
|
||||
t.Errorf("failed to preload: %v", err)
|
||||
return
|
||||
}
|
||||
main.NoStats()
|
||||
|
||||
defer func() {
|
||||
client := main.PilosaClient()
|
||||
if client == nil {
|
||||
return
|
||||
}
|
||||
if err := client.DeleteIndexByName(main.Pilosa.Index); err != nil {
|
||||
t.Errorf("deleting test index: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := main.Run(); err != nil {
|
||||
t.Errorf("failed to ingest: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
client := main.PilosaClient()
|
||||
|
||||
schema, err := client.Schema()
|
||||
if err != nil {
|
||||
t.Errorf("getting schema: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
index := schema.Indexes()[main.Pilosa.Index]
|
||||
if index == nil {
|
||||
t.Error("index dissapeared")
|
||||
return
|
||||
}
|
||||
|
||||
expectKeys := keyedIndices[src]
|
||||
indexKeys := index.Opts().Keys()
|
||||
if indexKeys != expectKeys {
|
||||
t.Errorf("expected keys=%v; got keys=%v", expectKeys, indexKeys)
|
||||
}
|
||||
|
||||
resp, err := client.Query(index.Count(index.All()))
|
||||
if err != nil {
|
||||
t.Errorf("validation query failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
recordCount := resp.Result().Count()
|
||||
// some sources have duplicate records with same id.
|
||||
// record count in featurebase won't match the number of ingested records
|
||||
if duplicateIndices[src] {
|
||||
if recordCount == 0 {
|
||||
t.Errorf("expected %d records; found %d records", 0, recordCount)
|
||||
}
|
||||
} else {
|
||||
if recordCount != n {
|
||||
t.Errorf("expected %d records; found %d records", n, recordCount)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigKafka(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
src string
|
||||
conc int
|
||||
numpartitions int
|
||||
replicationfactor int
|
||||
topic string
|
||||
batchsize int
|
||||
target string
|
||||
}{
|
||||
{
|
||||
src: sources[0],
|
||||
conc: 2,
|
||||
numpartitions: 4,
|
||||
replicationfactor: 2,
|
||||
topic: fmt.Sprintf("topic-%d", rand.New(rand.NewSource(time.Now().Unix())).Uint64()),
|
||||
batchsize: 1000000,
|
||||
target: TargetKafka,
|
||||
},
|
||||
{
|
||||
src: sources[0],
|
||||
conc: 1,
|
||||
numpartitions: 1,
|
||||
replicationfactor: 1,
|
||||
topic: fmt.Sprintf("topic-%d", rand.New(rand.NewSource(time.Now().Unix())).Uint64()),
|
||||
batchsize: 10000,
|
||||
target: TargetKafkaStatic,
|
||||
},
|
||||
{
|
||||
src: sources[0],
|
||||
conc: 3,
|
||||
numpartitions: 1,
|
||||
replicationfactor: 1,
|
||||
topic: fmt.Sprintf("topic-%d", rand.New(rand.NewSource(time.Now().Unix())).Uint64()),
|
||||
batchsize: 10000,
|
||||
target: TargetKafkaStatic,
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
tc := test
|
||||
t.Run(fmt.Sprintf("Test%d", i), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
main := NewMain()
|
||||
main.Source = tc.src
|
||||
main.Concurrency = tc.conc
|
||||
main.Target = tc.target
|
||||
main.Concurrency = tc.conc
|
||||
main.Kafka.BatchSize = tc.batchsize
|
||||
main.Kafka.Topic = tc.topic
|
||||
main.Kafka.ReplicationFactor = tc.replicationfactor
|
||||
main.Kafka.NumPartitions = tc.numpartitions
|
||||
err := main.Preload()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to preload: %v", err)
|
||||
}
|
||||
if main.KafkaPut.Concurrency != tc.conc {
|
||||
t.Fatalf("expected %d for concurrency, got %d", tc.conc, main.KafkaPut.Concurrency)
|
||||
}
|
||||
if main.KafkaPut.BatchSize != tc.batchsize {
|
||||
t.Fatalf("expected %d for batch-size, got %d", tc.batchsize, main.KafkaPut.BatchSize)
|
||||
}
|
||||
if main.KafkaPut.NumPartitions != tc.numpartitions {
|
||||
t.Fatalf("expected %d for num partitions, got %d", tc.numpartitions, main.KafkaPut.NumPartitions)
|
||||
}
|
||||
if main.KafkaPut.ReplicationFactor != tc.replicationfactor {
|
||||
t.Fatalf("expected %d for replication factor, got %d", tc.replicationfactor, main.KafkaPut.ReplicationFactor)
|
||||
}
|
||||
if main.KafkaPut.Target != tc.target {
|
||||
t.Fatalf("expected %s for target, got %s", tc.target, main.KafkaPut.Target)
|
||||
}
|
||||
if main.KafkaPut.Topic != tc.topic {
|
||||
t.Fatalf("expected %s for topic, got %s", tc.topic, main.KafkaPut.Topic)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func configureKafkaTestFlags(main *Main) {
|
||||
main.Pilosa.Hosts = []string{"pilosa:10101"}
|
||||
main.Kafka.KafkaBootstrapServers = []string{"kafka:9092"}
|
||||
}
|
||||
|
||||
func TestIntegrationWithKafka(t *testing.T) {
|
||||
tests := []struct {
|
||||
target string
|
||||
numPartitions int
|
||||
primaryKeyFields []string
|
||||
idField string
|
||||
}{
|
||||
{
|
||||
target: TargetKafka,
|
||||
numPartitions: 6,
|
||||
primaryKeyFields: []string{"uuid"},
|
||||
idField: "uuid",
|
||||
},
|
||||
{
|
||||
target: TargetKafka,
|
||||
numPartitions: 1,
|
||||
primaryKeyFields: []string{},
|
||||
idField: "",
|
||||
},
|
||||
{
|
||||
target: TargetKafkaStatic,
|
||||
numPartitions: 4,
|
||||
primaryKeyFields: []string{},
|
||||
idField: "",
|
||||
},
|
||||
{
|
||||
target: TargetKafkaStatic,
|
||||
numPartitions: 2,
|
||||
primaryKeyFields: []string{"uuid"},
|
||||
idField: "uuid",
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
tc := test
|
||||
ti := i
|
||||
t.Run(tc.target, func(t *testing.T) {
|
||||
const n = 100
|
||||
src := "custom"
|
||||
|
||||
main := NewMain()
|
||||
main.Pilosa.Index = fmt.Sprintf("test-%s-%d-%d", tc.target, ti, rand.New(rand.NewSource(time.Now().Unix())).Uint64())
|
||||
main.Source = src
|
||||
main.CustomConfig = "testdata/tremor_keys.yaml"
|
||||
main.Target = tc.target
|
||||
main.EndAt = n - 1
|
||||
main.Pilosa.BatchSize = 13
|
||||
configureKafkaTestFlags(main)
|
||||
main.Kafka.Topic = fmt.Sprintf("topic-%s-%d-%d", tc.target, ti, rand.New(rand.NewSource(time.Now().Unix())).Uint64())
|
||||
main.Kafka.NumPartitions = tc.numPartitions
|
||||
main.Concurrency = tc.numPartitions // make concurrency and partitions equal
|
||||
main.Kafka.ReplicationFactor = 1
|
||||
if tc.target == TargetKafka {
|
||||
main.Kafka.SchemaRegistryURL = "http://schema-registry:8081"
|
||||
}
|
||||
if len(tc.primaryKeyFields) != 0 {
|
||||
main.idkMain.PrimaryKeyFields = tc.primaryKeyFields
|
||||
}
|
||||
if tc.idField != "" {
|
||||
main.idkMain.IDField = tc.idField
|
||||
}
|
||||
if err := main.Preload(); err != nil {
|
||||
t.Fatalf("failed to preload: %v", err)
|
||||
}
|
||||
|
||||
if err := main.KafkaPut.Run(); err != nil {
|
||||
t.Fatalf("failed to ingest records to kafka: %v", err)
|
||||
}
|
||||
|
||||
if err := main.KafkaPut.ConfigMap.SetKey("go.produce.channel.size", 1); err != nil {
|
||||
t.Fatalf("failed to st produce channel size: %v", err)
|
||||
}
|
||||
|
||||
confluentCommand := idk.ConfluentCommand{
|
||||
KafkaBootstrapServers: []string{"kafka:9092"},
|
||||
KafkaSocketTimeoutMs: 10,
|
||||
}
|
||||
|
||||
configMap, err := common.SetupConfluent(&confluentCommand)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create kafka confluent config map: %v", err)
|
||||
}
|
||||
|
||||
kafkaClient, err := kafka.NewAdminClient(configMap)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get kafka client: %v", err)
|
||||
}
|
||||
defer kafkaClient.Close()
|
||||
|
||||
metadata, err := kafkaClient.GetMetadata(&main.KafkaPut.Topic, false, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get kafka metadata: %v", err)
|
||||
}
|
||||
|
||||
topic, ok := metadata.Topics[main.KafkaPut.Topic]
|
||||
if !ok {
|
||||
t.Fatalf("topic %s not found in metadata", main.KafkaPut.Topic)
|
||||
}
|
||||
|
||||
if len(topic.Partitions) != tc.numPartitions {
|
||||
t.Fatalf("expected %d for num partitions, got %d", tc.numPartitions, len(topic.Partitions))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartEnds(t *testing.T) {
|
||||
tests := []struct {
|
||||
config SourceGeneratorConfig
|
||||
conc int
|
||||
ses []startEnd
|
||||
total uint64
|
||||
}{
|
||||
{
|
||||
config: SourceGeneratorConfig{
|
||||
StartFrom: 0,
|
||||
EndAt: 100,
|
||||
Concurrency: 5},
|
||||
conc: 5,
|
||||
ses: []startEnd{{0, 20}, {20, 40}, {40, 60}, {60, 80}, {80, 100}},
|
||||
total: 100,
|
||||
},
|
||||
{
|
||||
config: SourceGeneratorConfig{
|
||||
StartFrom: 0,
|
||||
EndAt: 100,
|
||||
Concurrency: 1},
|
||||
conc: 1,
|
||||
ses: []startEnd{{0, 100}},
|
||||
total: 100,
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
tc := test
|
||||
t.Run(fmt.Sprintf("Test_%d", i), func(t *testing.T) {
|
||||
conc, ses, total := startEnds(tc.config)
|
||||
if conc != tc.conc {
|
||||
t.Fatalf("expected %d for concurrency, got %d", tc.conc, conc)
|
||||
}
|
||||
if total != tc.total {
|
||||
t.Fatalf("expected %d for total records, got %d", tc.total, total)
|
||||
}
|
||||
if len(ses) != len(tc.ses) {
|
||||
t.Fatalf("expected %d of boundaries, got %d", len(tc.ses), len(ses))
|
||||
}
|
||||
for i, exp := range tc.ses {
|
||||
got := ses[i]
|
||||
if exp.start != got.start || exp.end != got.end {
|
||||
t.Fatalf("expected boundaries to start %d and end at %d, but got start %d and end at %d", exp.start, exp.end, got.start, got.end)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
22
idk/datagen/dell.data.go
Normal file
22
idk/datagen/dell.data.go
Normal file
File diff suppressed because one or more lines are too long
171
idk/datagen/dell.go
Normal file
171
idk/datagen/dell.go
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
)
|
||||
|
||||
// Ensure Dell implements interface.
|
||||
var _ Sourcer = (*Dell)(nil)
|
||||
|
||||
// Dell implements Sourcer, and returns a very basic
|
||||
// data set. It can be used as an Dell for writing
|
||||
// additional custom Sourcers.
|
||||
type Dell struct{}
|
||||
|
||||
// NewDell returns a new instance of Dell.
|
||||
func NewDell(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Dell{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (e *Dell) Source(cfg SourceConfig) idk.Source {
|
||||
src := &DellSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
rand: rand.New(rand.NewSource(19)),
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "service_tag"}, // 1
|
||||
idk.IntField{NameVal: "record_date", Min: intptr(10000), Max: intptr(30000)}, // 2
|
||||
idk.StringField{NameVal: "system_model"}, // 3
|
||||
idk.StringField{NameVal: "bios_version"}, // 4
|
||||
idk.StringField{NameVal: "operating_system"}, // 5
|
||||
idk.IntField{NameVal: "system_ram_gb", Min: intptr(1), Max: intptr(256)}, // 6
|
||||
idk.IntField{NameVal: "video_fan_speed", Min: intptr(0), Max: intptr(20)}, // 7
|
||||
idk.IntField{NameVal: "proc_fan_speed", Min: intptr(0), Max: intptr(20)}, // 8
|
||||
idk.StringField{NameVal: "epsa_fail_code"}, // 9
|
||||
idk.StringField{NameVal: "epsa_version"}, // 10
|
||||
idk.StringField{NameVal: "video_controller"}, // 11
|
||||
idk.StringField{NameVal: "ddv_revision"}, // 12
|
||||
idk.StringField{NameVal: "motherboard_eppid"}, // 13
|
||||
idk.IntField{NameVal: "proc_fan_avg", Min: intptr(0), Max: intptr(30)}, // 14
|
||||
idk.StringField{NameVal: "proc_information"}, // 15
|
||||
},
|
||||
}
|
||||
|
||||
src.system_modelZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(system_model))-1)
|
||||
src.operating_systemZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(operating_system))-1)
|
||||
src.system_ramZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(system_ram))-1)
|
||||
src.epsa_fail_codeZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(epsa_fail_code))-1)
|
||||
src.epsa_versionZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(epsa_version))-1)
|
||||
src.video_controllerZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(video_controller))-1)
|
||||
src.ddv_revisionZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(ddv_revision))-1)
|
||||
src.motherboard_eppidZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(motherboard_eppid))-1)
|
||||
src.processor_informationZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(processor_information))-1)
|
||||
src.bios_versionZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(bios_version))-1)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (e *Dell) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (e *Dell) DefaultEndAt() uint64 {
|
||||
return 100
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (e *Dell) Info() string {
|
||||
return "Generates data representative of Dell's internal hardware tracking operations."
|
||||
}
|
||||
|
||||
// Ensure DellSource implements interface.
|
||||
var _ idk.Source = (*DellSource)(nil)
|
||||
|
||||
// DellSource is an instance of a source generated
|
||||
// by the Sourcer implementation Dell.
|
||||
type DellSource struct {
|
||||
cur uint64
|
||||
endAt uint64
|
||||
|
||||
rand *rand.Rand
|
||||
system_modelZipf *rand.Zipf
|
||||
operating_systemZipf *rand.Zipf
|
||||
system_ramZipf *rand.Zipf
|
||||
epsa_fail_codeZipf *rand.Zipf
|
||||
epsa_versionZipf *rand.Zipf
|
||||
video_controllerZipf *rand.Zipf
|
||||
ddv_revisionZipf *rand.Zipf
|
||||
motherboard_eppidZipf *rand.Zipf
|
||||
processor_informationZipf *rand.Zipf
|
||||
bios_versionZipf *rand.Zipf
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
// Record implements idk.Source.
|
||||
func (e *DellSource) Record() (idk.Record, error) {
|
||||
if e.cur >= e.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
e.record[0] = e.cur
|
||||
e.record[1] = e.StringDell(7)
|
||||
e.record[2] = e.generateRandomInt(30000, 10000)
|
||||
e.record[3] = system_model[e.system_modelZipf.Uint64()]
|
||||
e.record[4] = bios_version[e.bios_versionZipf.Uint64()]
|
||||
e.record[5] = operating_system[e.operating_systemZipf.Uint64()]
|
||||
e.record[6] = system_ram[e.system_ramZipf.Uint64()]
|
||||
e.record[7] = e.generateRandomInt(20, 0)
|
||||
e.record[8] = e.generateRandomInt(20, 0)
|
||||
e.record[9] = epsa_fail_code[e.epsa_fail_codeZipf.Uint64()]
|
||||
e.record[10] = epsa_version[e.epsa_versionZipf.Uint64()]
|
||||
e.record[11] = video_controller[e.video_controllerZipf.Uint64()]
|
||||
e.record[12] = ddv_revision[e.ddv_revisionZipf.Uint64()]
|
||||
e.record[13] = motherboard_eppid[e.motherboard_eppidZipf.Uint64()]
|
||||
e.record[14] = e.generateRandomInt(30, 0)
|
||||
e.record[15] = processor_information[e.processor_informationZipf.Uint64()]
|
||||
|
||||
e.cur++
|
||||
return e.record, nil
|
||||
}
|
||||
|
||||
// Schema implements idk.Source.
|
||||
func (e *DellSource) Schema() []idk.Field {
|
||||
return e.schema
|
||||
}
|
||||
func (e *DellSource) Seed(seed int64) {
|
||||
e.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*DellSource)(nil)
|
||||
|
||||
const charsetDell = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
|
||||
// returns a string with random characters from a predefined charset of a specified length
|
||||
func (e *DellSource) StringWithCharset(length int, charset string) string {
|
||||
b := make([]byte, length)
|
||||
for i := range b {
|
||||
b[i] = charsetDell[e.rand.Intn(len(charsetDell))]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
//returns a string with random English alphabets of the specified length
|
||||
func (e *DellSource) StringDell(length int) string {
|
||||
return e.StringWithCharset(length, charsetDell)
|
||||
}
|
||||
|
||||
//generates a random int between range max and min (inclusive)
|
||||
func (e *DellSource) generateRandomInt(max int, min int) int {
|
||||
return e.rand.Intn(max-min) + min
|
||||
}
|
||||
|
||||
func (e *DellSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
124
idk/datagen/dwarranty.go
Normal file
124
idk/datagen/dwarranty.go
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
)
|
||||
|
||||
// Ensure DWarranty implements interface.
|
||||
var _ Sourcer = (*DWarranty)(nil)
|
||||
|
||||
// DWarranty implements Sourcer.
|
||||
type DWarranty struct{}
|
||||
|
||||
// NewDWarranty returns a new instance of DWarranty.
|
||||
func NewDWarranty(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &DWarranty{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (d *DWarranty) Source(cfg SourceConfig) idk.Source {
|
||||
min := int64(0)
|
||||
src := &DWarrantySource{
|
||||
totalGenerating: cfg.total,
|
||||
WarrantySource: WarrantySource{
|
||||
startFrom: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
rand: rand.New(rand.NewSource(19)),
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"},
|
||||
idk.StringField{NameVal: "type"},
|
||||
idk.IntField{NameVal: "start", Min: &min},
|
||||
idk.IntField{NameVal: "last", Min: &min},
|
||||
idk.IntField{NameVal: "cust_id", Min: &min},
|
||||
idk.IntField{NameVal: "item_id", Min: &min},
|
||||
idk.StringField{NameVal: "prod"},
|
||||
},
|
||||
},
|
||||
}
|
||||
src.typeZipf = rand.NewZipf(src.rand, 1.03, 5, uint64(len(types)-1))
|
||||
src.prodZipf = rand.NewZipf(src.rand, 1.2, 8, uint64(len(prods)-1))
|
||||
src.custZipf = rand.NewZipf(src.rand, 1.01, 4, 100000000)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[2] = make([]string, 52*5)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (d *DWarranty) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (d *DWarranty) DefaultEndAt() uint64 {
|
||||
return 2147483648
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (d *DWarranty) Info() string {
|
||||
return "TODO"
|
||||
}
|
||||
|
||||
// Ensure DWarrantySource implements interface.
|
||||
var _ idk.Source = (*DWarrantySource)(nil)
|
||||
|
||||
type DWarrantySource struct {
|
||||
WarrantySource
|
||||
|
||||
totalGenerating uint64
|
||||
prodZipf *rand.Zipf
|
||||
}
|
||||
|
||||
func (s *DWarrantySource) Record() (idk.Record, error) {
|
||||
if s.startFrom >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.startFrom
|
||||
s.record[1] = types[s.typeZipf.Uint64()]
|
||||
startDay := (s.startFrom * uint64(daysSinceStart)) / s.totalGenerating
|
||||
days := numDays(s.rand)
|
||||
s.record[2] = startDay
|
||||
s.record[3] = startDay + uint64(days)
|
||||
s.record[4] = int64(s.custZipf.Uint64()) // 100M customers
|
||||
s.record[5] = s.rand.Intn(1500000000) // 1.5B items
|
||||
s.record[6] = prods[s.prodZipf.Uint64()]
|
||||
s.startFrom++
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *DWarrantySource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
var _ Seedable = (*DWarrantySource)(nil)
|
||||
|
||||
var types = make([]string, 100)
|
||||
var prods = make([]string, 3200)
|
||||
|
||||
func init() {
|
||||
var baseTypes = []string{"AZ-PQ-SUP1", "AZ-PQ-GOLDSUP", "BASICSUPPORT", "LM-NO-BASICLOWSUP", "QZ-PG-NONE", "CLIENTFGBG", "PHONE-BUNDLE", "AZ-PQ-PLATSUP", "AZ-PQ-PARTSUP", "OTHERSUP"}
|
||||
for i := range types {
|
||||
bt := baseTypes[i%len(baseTypes)]
|
||||
types[i] = fmt.Sprintf("%s-%d", bt, i)
|
||||
}
|
||||
|
||||
for i := range prods {
|
||||
prods[i] = fmt.Sprintf("PRD-%d", i)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DWarrantySource) Close() error {
|
||||
return nil
|
||||
}
|
||||
269
idk/datagen/equipment.data.go
Normal file
269
idk/datagen/equipment.data.go
Normal file
File diff suppressed because one or more lines are too long
282
idk/datagen/equipment.go
Normal file
282
idk/datagen/equipment.go
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Equipment implements interface.
|
||||
var _ Sourcer = (*Equipment)(nil)
|
||||
|
||||
// Equipment implements Sourcer.
|
||||
type Equipment struct{}
|
||||
|
||||
// NewEquipment returns a new instance of Equipment.
|
||||
func NewEquipment(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Equipment{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (e *Equipment) Source(cfg SourceConfig) idk.Source {
|
||||
src := &EquipmentSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "domain"}, // 1
|
||||
idk.IntField{NameVal: "cost"}, // 2
|
||||
idk.IntField{NameVal: "site_id"}, // 3
|
||||
idk.DateIntField{NameVal: "date_acquired", Unit: idk.Day}, // 4
|
||||
idk.DateIntField{NameVal: "last_maintenance", Unit: idk.Day}, // 5
|
||||
idk.StringField{NameVal: "name"}, // 6
|
||||
idk.StringField{NameVal: "type"}, // 7
|
||||
idk.StringField{NameVal: "manufacturer"}, // 8
|
||||
idk.StringField{NameVal: "model"}, // 9
|
||||
},
|
||||
mfgrZipfs: make(map[string]map[string]*rand.Zipf),
|
||||
modelZipfs: make(map[string]map[string]map[string]*rand.Zipf),
|
||||
}
|
||||
|
||||
src.dcZipf = rand.NewZipf(src.rand, 1.5, 6, 3000000)
|
||||
|
||||
// set up zipfs for all equipment manufacturers and models
|
||||
//
|
||||
// TODO: I believe the fact that we make rand calls during map
|
||||
// iteration (where order is random) is making this datagen
|
||||
// non-reproducible.
|
||||
for dom, typeMap := range domains {
|
||||
src.mfgrZipfs[dom] = make(map[string]*rand.Zipf)
|
||||
src.modelZipfs[dom] = make(map[string]map[string]*rand.Zipf)
|
||||
for typ, mfgrMap := range typeMap {
|
||||
src.modelZipfs[dom][typ] = make(map[string]*rand.Zipf)
|
||||
src.mfgrZipfs[dom][typ] = rand.NewZipf(src.rand, 1.001+src.rand.Float64(), float64(src.rand.Intn(7))+src.rand.Float64()+1, uint64(len(mfgrMap))-1)
|
||||
for mfgr, models := range mfgrMap {
|
||||
src.modelZipfs[dom][typ][mfgr] = rand.NewZipf(src.rand, 1.001+src.rand.Float64(), float64(src.rand.Intn(7))+src.rand.Float64()+1, uint64(len(models))-1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (e *Equipment) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (e *Equipment) DefaultEndAt() uint64 {
|
||||
return 36000000 - 1
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (e *Equipment) Info() string {
|
||||
return "Generates equipment and device data (manufacturer, cost, etc) - includes a foreign key for network_ts, power_ts, transaction_ts."
|
||||
}
|
||||
|
||||
// Ensure EquipmentSource implements interface.
|
||||
var _ idk.Source = (*EquipmentSource)(nil)
|
||||
|
||||
type EquipmentSource struct {
|
||||
Log logger.Logger
|
||||
|
||||
rand *rand.Rand
|
||||
mfgrZipfs map[string]map[string]*rand.Zipf
|
||||
modelZipfs map[string]map[string]map[string]*rand.Zipf
|
||||
dcZipf *rand.Zipf
|
||||
schema []idk.Field
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
record record
|
||||
}
|
||||
|
||||
func (s *EquipmentSource) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.cur
|
||||
switch {
|
||||
case s.cur < 5000000:
|
||||
s.fillRecord("Power", s.powerType())
|
||||
case s.cur < 35000000:
|
||||
s.fillRecord("Communications", s.communicationsType())
|
||||
case s.cur < 36000000:
|
||||
s.fillRecord("Transactions", s.transactionsType())
|
||||
default:
|
||||
return nil, errors.New("exhausted equipment data set")
|
||||
}
|
||||
s.cur++
|
||||
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *EquipmentSource) fillRecord(domain, eqType string) {
|
||||
s.record[1] = domain
|
||||
|
||||
// type
|
||||
s.record[7] = eqType
|
||||
|
||||
// manufacturer
|
||||
mfgrs := mfgrLists[domain][eqType]
|
||||
mfgr := mfgrs[s.mfgrZipfs[domain][eqType].Uint64()]
|
||||
s.record[8] = mfgr
|
||||
|
||||
// model
|
||||
models := domains[domain][eqType][mfgr]
|
||||
modelZipf, ok := s.modelZipfs[domain][eqType][mfgr]
|
||||
if !ok {
|
||||
s.Log.Printf("%s %s %s", domain, eqType, mfgr)
|
||||
}
|
||||
model := models[modelZipf.Uint64()]
|
||||
s.record[9] = model
|
||||
|
||||
// cost
|
||||
s.record[2] = costs[domain][eqType].Cost(s.rand)
|
||||
|
||||
// site id
|
||||
s.record[3] = s.siteID(eqType)
|
||||
|
||||
// date acquired
|
||||
s.record[4] = startDayInt + s.rand.Int63n(365*10-200)
|
||||
|
||||
// last maintenance
|
||||
s.record[5] = s.record[4].(int64) + s.rand.Int63n(todayInt-s.record[4].(int64))
|
||||
|
||||
}
|
||||
|
||||
func (s *EquipmentSource) powerType() string {
|
||||
num := s.rand.Intn(100)
|
||||
switch {
|
||||
case num < 75:
|
||||
return "Breaker"
|
||||
case num < 77:
|
||||
return "Generator"
|
||||
case num < 82:
|
||||
return "Air Handler"
|
||||
case num < 88:
|
||||
return "Chiller"
|
||||
default:
|
||||
return "UPS"
|
||||
}
|
||||
}
|
||||
|
||||
func (s *EquipmentSource) siteID(typ string) int64 {
|
||||
switch typ {
|
||||
case "Breaker":
|
||||
return s.rand.Int63n(4680000)
|
||||
case "Generator":
|
||||
return s.rand.Int63n(4680000)
|
||||
case "Air Handler":
|
||||
return s.rand.Int63n(4680000)
|
||||
case "Chiller":
|
||||
return s.rand.Int63n(4680000)
|
||||
case "UPS":
|
||||
return s.rand.Int63n(4680000)
|
||||
case "Router":
|
||||
num := s.rand.Int63n(1000 + 7000 + 3000000)
|
||||
switch {
|
||||
case num < 8000:
|
||||
// MSC or Headend
|
||||
return num
|
||||
default:
|
||||
// datacenter
|
||||
return num + 1680000
|
||||
}
|
||||
case "Server":
|
||||
num := s.rand.Intn(100)
|
||||
// not a datacenter
|
||||
if num == 0 {
|
||||
// non-DC
|
||||
return s.rand.Int63n(1688000)
|
||||
} else {
|
||||
// DC
|
||||
return int64(s.dcZipf.Uint64()) + 1688000
|
||||
}
|
||||
case "BTS":
|
||||
// Cell Site
|
||||
return s.rand.Int63n(400000) + 488000
|
||||
case "CPE":
|
||||
// Retail
|
||||
return s.rand.Int63n(400000) + 88000
|
||||
case "POS":
|
||||
// Retail
|
||||
return s.rand.Int63n(400000) + 88000
|
||||
default:
|
||||
panic(fmt.Sprintf("unknown type %s", typ))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *EquipmentSource) communicationsType() string {
|
||||
// 30M communications eq from 5M to 35M
|
||||
// 1M router, 14M Server, 13M CPE, 2M BTS
|
||||
id := s.record[0].(uint64)
|
||||
switch {
|
||||
case id < 18000000:
|
||||
return "CPE"
|
||||
case id < 19000000:
|
||||
return "Router"
|
||||
case id < 33000000:
|
||||
return "Server"
|
||||
default:
|
||||
return "BTS"
|
||||
}
|
||||
}
|
||||
|
||||
func (s *EquipmentSource) transactionsType() string { return "POS" }
|
||||
|
||||
func (s *EquipmentSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *EquipmentSource) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*EquipmentSource)(nil)
|
||||
|
||||
var mfgrLists = make(map[string]map[string][]string)
|
||||
|
||||
func init() {
|
||||
for name, typeMap := range domains {
|
||||
mfgrLists[name] = make(map[string][]string)
|
||||
for typ, mfgrMap := range typeMap {
|
||||
mfgrs := make([]string, 0)
|
||||
for mfgr := range mfgrMap {
|
||||
mfgrs = append(mfgrs, mfgr)
|
||||
}
|
||||
mfgrLists[name][typ] = mfgrs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type minMax struct {
|
||||
min int64
|
||||
max int64
|
||||
}
|
||||
|
||||
func (m minMax) Cost(r *rand.Rand) int64 {
|
||||
return r.Int63n(m.max-m.min) + m.min
|
||||
|
||||
}
|
||||
|
||||
func (s *EquipmentSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
100
idk/datagen/example.go
Normal file
100
idk/datagen/example.go
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
)
|
||||
|
||||
// Ensure Example implements interface.
|
||||
var _ Sourcer = (*Example)(nil)
|
||||
|
||||
// Example implements Sourcer, and returns a very basic
|
||||
// data set. It can be used as an example for writing
|
||||
// additional custom Sourcers.
|
||||
type Example struct{}
|
||||
|
||||
// NewExample returns a new instance of Example.
|
||||
func NewExample(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Example{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
// NOTE: avro Name fields can only contain certain
|
||||
// characters.
|
||||
// See: https://avro.apache.org/docs/current/spec.html#names
|
||||
func (e *Example) Source(cfg SourceConfig) idk.Source {
|
||||
src := &ExampleSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "string"}, // 1
|
||||
},
|
||||
}
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[0] = uint64(0)
|
||||
src.record[1] = make([]byte, 12)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (e *Example) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (e *Example) DefaultEndAt() uint64 {
|
||||
return 100
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (e *Example) Info() string {
|
||||
return "generate data for a couple of fields to use as an example"
|
||||
}
|
||||
|
||||
// Ensure ExampleSource implements interface.
|
||||
var _ idk.Source = (*ExampleSource)(nil)
|
||||
|
||||
// ExampleSource is an instance of a source generated
|
||||
// by the Sourcer implementation Example.
|
||||
type ExampleSource struct {
|
||||
cur uint64
|
||||
endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
// Record implements idk.Source.
|
||||
func (e *ExampleSource) Record() (idk.Record, error) {
|
||||
if e.cur >= e.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
// Increment the ID.
|
||||
e.record[0] = e.cur
|
||||
e.record[1] = []byte(fmt.Sprintf("foo-%d", e.cur))
|
||||
e.cur++
|
||||
|
||||
return e.record, nil
|
||||
}
|
||||
|
||||
// Schema implements idk.Source.
|
||||
func (e *ExampleSource) Schema() []idk.Field {
|
||||
return e.schema
|
||||
}
|
||||
|
||||
func (e *ExampleSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
65
idk/datagen/gen/constants.data.go
Normal file
65
idk/datagen/gen/constants.data.go
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
package gen
|
||||
|
||||
var Animals = []string{"Millipede", "Reindeer", "Otter", "Umbrellabird", "Bandicoot", "Macaroni Penguin", "Australian Cattle Dog", "Hercules Beetle", "Magpie", "Bernese Mountain Dog",
|
||||
"Royal Penguin", "Manatee", "Black Russian Terrier", "Badger", "Indian Elephant", "Indochinese Tiger", "Centipede", "Bluetick Coonhound", "Humpback Whale", "Caiman Lizard",
|
||||
"Grasshopper", "Tortoise", "Alpine Dachsbracke", "Quetzal", "Aldabra Giant Tortoise", "Electric Eel", "Vervet Monkey", "Mountain Lion", "Geoffroys Tamarin", "Collared Peccary",
|
||||
"Doberman Pinscher", "Megalodon", "Insect", "Shiba Inu", "Tiger Shark", "Aye Aye", "Black Rhinoceros", "Gar", "Stick Insect", "Marine Toad",
|
||||
"Grouse", "Javanese", "Water Dragon", "Whippet", "Giant Clam", "Caiman", "Fennec Fox", "Weasel", "Rat", "Golden Retriever Complete Pet Guide",
|
||||
"Starfish", "Moth", "Horned Frog", "Water Vole", "King Crab", "Common Frog", "Neanderthal", "Fossa", "Frigatebird", "Spider Monkey",
|
||||
"Appenzeller Dog", "Emperor Penguin", "Saint Bernard", "Eagle", "Platypus", "Monkfish", "Lizard", "Barracuda", "Wombat", "Pika",
|
||||
"Yak", "Ibis", "Beetle", "Horse", "Arctic Fox", "Chihuahua", "African Wild Dog", "Bat", "Dogue De Bordeaux", "Akita",
|
||||
"Mastiff", "Fox Terrier", "Thorny Devil", "Dachshund", "Zonkey", "Dormouse", "Guinea Pig", "Edible Frog", "Barnacle", "Asian Palm Civet",
|
||||
"Irish WolfHound", "Giant Panda Bear", "Abyssinian", "Toucan", "Wild Boar", "Grizzly Bear", "Borneo Elephant", "Dusky Dolphin", "Gecko", "Golden Lion Tamarin",
|
||||
"Deer", "Indri", "Stoat", "Skunk", "Sea Lion", "Turkey", "Octopus", "Pelican", "Meerkat", "Pig",
|
||||
"Glass Lizard", "Fur Seal", "Pekingese", "Bottlenose Dolphin", "Fox", "Siamese", "Scorpion Fish", "Howler Monkey", "Anatolian Shepherd Dog", "Mountain Gorilla",
|
||||
"Burrowing Frog", "African Bush Elephant", "Havanese", "Sumatran Elephant", "Glow Worm", "Saber-Toothed Tiger", "Japanese Chin", "Tuatara", "Estrela Mountain Dog", "Cow",
|
||||
"Dwarf Crocodile", "Flounder", "Tetra", "Gorilla", "Bearded Collie", "Angelfish", "Gentoo Penguin", "Afghan Hound", "Porpoise", "Australian Kelpie Dog",
|
||||
"Pygmy Marmoset", "Harpy Eagle", "Ostrich", "Woolly Mammoth", "Duck", "Binturong", "Sea Slug", "Sumatran Tiger", "Cassowary", "King Penguin",
|
||||
"Bongo", "Basset Hound", "Siberian Husky", "Camel Spider", "Boxer Dog", "Cockroach", "Antelope", "Foxhound", "Beagle", "Impala",
|
||||
"Snapping Turtle", "Grey Reef Shark", "Jellyfish", "Pheasant", "Cavalier King Charles Spaniel", "Monarch Butterfly", "Wrasse", "Drever", "Catfish", "Malayan Civet",
|
||||
"Bearded Dragon", "Red Wolf", "Cuttlefish", "Chipmunk", "English Springer Spaniel", "Tiger Salamander", "Epagneul Pont Audemer", "Mongoose", "Somali", "Parrot",
|
||||
"Burmese", "Greyhound", "Tiger", "Tree Frog", "Skate Fish", "Flamingo", "Bonobo", "Ibizan Hound", "Spadefoot Toad", "Japanese Macaque",
|
||||
"Leopard Seal", "Leopard Tortoise", "Moose", "Chicken", "Avocet", "Australian Mist", "Little Penguin", "Quail", "Elephant Seal", "English Cocker Spaniel",
|
||||
"Humboldt Penguin", "Bedlington Terrier", "Blue Whale", "Tibetan Mastiff", "Pink Fairy Armadillo", "Hedgehog", "Bavarian Mountain Hound", "Basking Shark", "Quoll", "West Highland Terrier",
|
||||
"Polar Bear", "Egyptian Mau", "Fish", "Crocodile", "Vulture", "Arctic Wolf", "Javan Rhinoceros", "Capybara", "Baboon", "Gharial",
|
||||
"Hawaiian Crow", "Piranha", "Australian Shepherd", "Fishing Cat", "Sponge", "Guppy", "Tasmanian Devil", "Chesapeake Bay Retriever", "Lobster", "Elephant",
|
||||
"Labradoodle – The Complete Guide For Owners", "Radiated Tortoise", "Dugong", "French Bulldog", "Slow Worm", "Bulldog", "Giant African Land Snail", "African Tree Toad", "Serval", "Nurse Shark",
|
||||
"Kangaroo", "Jackal", "Neapolitan Mastiff", "Cockatoo", "Pond Skater", "Okapi", "Hyena", "Border Terrier", "Silver Dollar", "Dog",
|
||||
"Coati", "Butterfly", "Moorhen", "Numbat", "Human", "King Cobra", "Green Bee-Eater", "Common Loon", "Cross River Gorilla", "Moray Eel",
|
||||
"Bloodhound", "Spixs Macaw", "Sperm Whale", "Russian Blue", "Persian", "Chimpanzee", "Tiffany", "African Civet", "Koala", "Tapir",
|
||||
"Minke Whale", "Clumber Spaniel", "Leopard", "Liger", "Red Panda", "Wolf Spider", "Mayfly", "Zebra", "Bombay", "Rattlesnake",
|
||||
"African Forest Elephant", "Yorkshire Terrier", "Jack Russel", "Maltese", "Booby", "Bison", "Woodpecker", "Squid", "Uguisu", "Sumatran Orang-utan",
|
||||
"Asian Giant Hornet", "Pink Dolphin", "German Pinscher", "Dalmatian", "Sun Bear", "Caracal", "Camel", "Vampire Bat", "Mole", "Dolphin",
|
||||
"Nightingale", "German Shepherd Guide", "Coyote", "Mandrill", "Newfoundland", "Cairn Terrier", "Kingfisher", "Darwin’s Frog", "Mouse", "Aardvark",
|
||||
"Greater Swiss Mountain Dog", "Pied Tamarin", "Monkey", "Raccoon", "Sea Turtle", "Sloth", "Airedale Terrier", "Alligator", "Orang-utan", "Wyoming Toad",
|
||||
"Shrimp", "Steller’s Sea Cow", "Cotton-top Tamarin", "Asiatic Black Bear", "Western Lowland Gorilla", "Alaskan Malamute", "Caterpillar", "Spiny Dogfish", "Bull Terrier", "Maned Wolf",
|
||||
"Cocker Spaniel", "Buffalo", "Lemming", "Crane", "Tang", "Monitor Lizard", "Crab-Eating Macaque", "Kudu", "Dunker", "Border Collie",
|
||||
"Bobcat", "Grey Mouse Lemur", "South China Tiger", "Bolognese Dog", "Kiwi", "Horseshoe Crab", "Highland Cattle", "Stingray", "Macaw", "Patas Monkey",
|
||||
"Bornean Orang-utan", "Birman", "Emu", "Spectacled Bear", "Lionfish", "Golden Masked Owl", "Honey Badger", "Wildebeest", "Quokka", "Hippopotamus",
|
||||
"Heron", "Cichlid", "Water Buffalo", "Chinook", "African Penguin", "Common Toad", "Termite", "Affenpinscher", "Leaf-Tailed Gecko", "Maine Coon",
|
||||
"Lynx", "Bactrian Camel", "Water Spaniel", "Harrier", "Manta Ray", "Coral", "Dogo Argentino", "Gerbil", "Australian Terrier", "Masked Palm Civet",
|
||||
"Snake", "Desert Tortoise", "Axolotl", "Greenland Dog", "Indian Palm Squirrel", "Fluke Fish (summer flounder)", "Woolly Monkey", "Ladybug", "Wolf", "Common Buzzard",
|
||||
"African Palm Civet", "Cesky Fousek", "Mule", "Amur Leopard", "Norwegian Forest", "Flying Squirrel", "Eskimo Dog", "Monte Iberia Eleuth", "Lion", "Eastern Lowland Gorilla",
|
||||
"Blobfish", "Chamois", "Arctic Hare", "Grey Seal", "English Shepherd", "Swan", "Sea Urchin", "Birds Of Paradise", "Butterfly Fish", "Blue Jay",
|
||||
"Staffordshire Bull Terrier", "Siamese Fighting Fish", "White-Faced Capuchin", "Whale Shark", "Snail", "Pit Bull Terrier", "Squirrel Monkey", "Hoopoe", "Sea Dragon", "Seahorse",
|
||||
"Great Dane", "Clown Fish", "Snowy Owl", "Anteater", "White Rhinoceros", "Boykin Spaniel", "Green Anole", "Pug", "Dhole", "Goose",
|
||||
"Deutsche Bracke", "Banded Palm Civet", "Llama", "Sri Lankan Elephant", "Welsh Corgi: The Complete Pet Guide", "Tropicbird", "Rhinoceros", "Curly Coated Retriever", "Leopard Cat", "Black Widow Spider",
|
||||
"Ragdoll", "Horn Shark", "Donkey", "Elephant Shrew", "Armadillo", "Keel Billed Toucan", "Purple Emperor", "Scorpion", "Chameleon", "Sea Otter",
|
||||
"Giant Schnauzer", "Bear", "Honey Bee", "Zebu", "African Clawed Frog", "Blue Lacy Dog", "Pangolin", "Cat", "Collie", "Dodo",
|
||||
"Brown Bear", "Akbash", "Tapanuli Orang-utan", "Puma", "Iguana", "Uakari", "Chinese Crested Dog", "Magellanic Penguin", "Hare", "Molly",
|
||||
"Peacock", "Saola", "Squirrel", "Basenji Dog", "Bumblebee", "Ferret", "Staffordshire Terrier", "Ainu Dog", "Rottweiler", "Beaver",
|
||||
"Sparrow", "Sea Squirt", "Labrador Retriever", "Pool Frog", "Bichon Frise", "Oyster", "Panther", "Hammerhead Shark", "Frog", "Ocelot",
|
||||
"Coonhound", "Rabbit", "Adelie Penguin", "Olm", "Porcupine", "Hummingbird", "Wasp", "Tawny Owl", "Walrus", "X-Ray Tetra",
|
||||
"Goat", "Komodo Dragon", "Fly", "Western Gorilla", "Golden-Crowned Flying Fox", "Stag Beetle", "Budgerigar", "Chinchilla", "Finnish Spitz", "Woodlouse",
|
||||
"Bull Mastiff", "Jaguar", "River Turtle", "Wolverine", "Siberian Tiger", "Jerboa", "Crab", "Emperor Tamarin", "Kakapo", "Hermit Crab",
|
||||
"Eskimo Dog", "North American Black Bear", "Galapagos Penguin", "Chow Chow", "Red-handed Tamarin", "Albatross", "Roseate Spoonbill", "Siberian", "Striped Rocket Frog", "Malayan Tiger",
|
||||
"Boston Terrier: Complete Pet Guide", "Poodle", "Clouded Leopard", "Red Knee Tarantula", "Yellow-Eyed Penguin", "Zebra Shark", "Lemur", "Raccoon Dog", "Pointer", "Newt",
|
||||
"Dragonfly", "Carolina Dog", "Xerus", "Crested Penguin", "Warthog", "Ermine", "Frilled Lizard", "Field Spaniel", "Himalayan", "Turkish Angora",
|
||||
"Discus", "Echidna", "Markhor", "Pygmy Hippopotamus", "Fin Whale", "Canaan Dog", "Marsh Frog", "Gopher", "Bull Shark", "Salamander",
|
||||
"Brazilian Terrier", "Balinese", "Asian Elephant", "Prawn", "Robin", "Entlebucher Mountain Dog", "Snowshoe", "Galapagos Tortoise", "Barn Owl", "Proboscis Monkey",
|
||||
"Wallaby", "Gila Monster", "Indian Rhinoceros", "Bullfrog", "Norfolk Terrier", "Sumatran Rhinoceros", "Bengal Tiger", "Penguin", "Mongrel", "Cougar",
|
||||
"Narwhal", "Old English Sheepdog", "Flat Coat Retriever", "Pere Davids Deer", "Eastern Gorilla", "Barb", "Golden Oriole", "Pademelon", "Sheep", "Indian Star Tortoise",
|
||||
"Aurochs", "Possum", "Zorse", "Chinstrap Penguin", "Sucker Fish", "Great White Shark", "Gibbon", "Guinea Fowl", "Sand Lizard", "Earwig",
|
||||
"Seal", "Irish Setter Complete Pet Guide", "Rock Hyrax", "Cheetah", "Puffin", "American Bulldog", "Pufferfish", "White Tiger", "Hamster", "Rockhopper Penguin",
|
||||
"Shih Tzu", "Fire-Bellied Toad", "Cuscus", "Pike", "Poison Dart Frog", "Dingo", "Puss Moth", "Ant", "Tarsier", "Scimitar-horned Oryx",
|
||||
"Long-Eared Owl", "Falcon", "Killer Whale", "Bird", "Giraffe", "Opossum"}
|
||||
154
idk/datagen/gen/gen.go
Normal file
154
idk/datagen/gen/gen.go
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
package gen
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
)
|
||||
|
||||
type Gen struct {
|
||||
seed int64
|
||||
|
||||
// cache of zipfs for various imax. All have s==1.1 and v==10
|
||||
zipfs map[uint64]*rand.Zipf
|
||||
|
||||
R *rand.Rand
|
||||
}
|
||||
|
||||
type Opt func(g *Gen)
|
||||
|
||||
// OptGenSeed is a functional option for providing a seed
|
||||
// to Gen.
|
||||
func OptGenSeed(s int64) Opt {
|
||||
return func(g *Gen) {
|
||||
g.seed = s
|
||||
}
|
||||
}
|
||||
|
||||
// New creates a new *Gen
|
||||
func New(opts ...Opt) *Gen {
|
||||
g := &Gen{
|
||||
seed: 17,
|
||||
zipfs: make(map[uint64]*rand.Zipf),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(g)
|
||||
}
|
||||
g.R = rand.New(rand.NewSource(g.seed))
|
||||
return g
|
||||
}
|
||||
|
||||
var alphaUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
var alphaLower = "abcdefghijklmnopqrstuvwxyz"
|
||||
|
||||
func (g *Gen) AlphaUpper(bs []byte) {
|
||||
for i := range bs {
|
||||
bs[i] = alphaUpper[g.R.Intn(len(alphaUpper))]
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Gen) AlphaLower(bs []byte) {
|
||||
for i := range bs {
|
||||
bs[i] = alphaLower[g.R.Intn(len(alphaLower))]
|
||||
}
|
||||
}
|
||||
|
||||
// StringSliceFromList repopulates input with a random number of items
|
||||
// from list. If input does not have enough capacity it may return a
|
||||
// new list. There may be repeat items, there may be 0 items, there
|
||||
// will never be more items than the length of list.
|
||||
func (g *Gen) StringSliceFromList(input []string, list []string) []string {
|
||||
input = input[:0]
|
||||
num := g.R.Intn(len(list) + 1)
|
||||
for i := 0; i < num; i++ {
|
||||
input = append(input, list[g.R.Intn(len(list))])
|
||||
}
|
||||
|
||||
return input
|
||||
}
|
||||
|
||||
// StringSliceFromListWeighted picks between 0 and upto items from list randomly, but zipfian weighted toward items earlier in list. It will try to re-use input, but may allocate a new list if needed.
|
||||
func (g *Gen) StringSliceFromListWeighted(input []string, list []string, s, v float64) []string {
|
||||
input = input[:0]
|
||||
z := g.getZipfParams(uint64(len(list)-1), s, v)
|
||||
num := z.Uint64()
|
||||
for i := uint64(0); i < num; i++ {
|
||||
input = append(input, list[z.Uint64()])
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
// StringFromListWeighted returns a random string from the list
|
||||
// weighted toward the items earlier in the list.
|
||||
func (g *Gen) StringFromListWeighted(list []string) string {
|
||||
z := g.getZipf(uint64(len(list) - 1))
|
||||
return list[z.Uint64()]
|
||||
}
|
||||
|
||||
// StringFromList returns a random string from the list
|
||||
// each with uniform probability
|
||||
func (g *Gen) StringFromList(list []string) string {
|
||||
return list[g.R.Intn(len(list))]
|
||||
}
|
||||
|
||||
// Set generates a random set of integers between min and max (inclusive).
|
||||
// The mean cardinality of the set is specified by typicalCardinality.
|
||||
func (g *Gen) Set(min uint64, max uint64, typicalCardinality uint64) map[uint64]struct{} {
|
||||
// The expected value of a negative binomial distribution is E(X) = r/p.
|
||||
// In this case we want E(x) = typicalCardinality.
|
||||
// In this context, a success means that we stop producing elements, so r = 1.
|
||||
// Therefore, p = 1/typicalCardinality.
|
||||
p := 1 / float64(typicalCardinality)
|
||||
|
||||
// Create a random target cardinality.
|
||||
var n uint64
|
||||
for n < max-min && g.R.Float64() > p {
|
||||
n++
|
||||
}
|
||||
|
||||
set := map[uint64]struct{}{}
|
||||
|
||||
if n > (max-min)/2 {
|
||||
// The target cardinality is a substantial portion of the entire range.
|
||||
// Generating random values and checking would result in many collisions and raise the complexity.
|
||||
// Generate a list of values, shuffle it, and select the first n entries.
|
||||
list := make([]uint64, max-min)
|
||||
for i := range list {
|
||||
list[i] = uint64(i) + min
|
||||
}
|
||||
g.R.Shuffle(len(list), func(i, j int) {
|
||||
list[j], list[i] = list[i], list[j]
|
||||
})
|
||||
list = list[:n]
|
||||
for _, v := range list {
|
||||
set[v] = struct{}{}
|
||||
}
|
||||
} else {
|
||||
// The target cardinality is a small portion of the range.
|
||||
// Randomly selecting values should be fairly fast since collisions should be somewhat infrequent.
|
||||
for uint64(len(set)) < n {
|
||||
// Generate another value for the set.
|
||||
set[min+uint64(g.R.Int63n(int64((max-min)+1)))] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
return set
|
||||
}
|
||||
|
||||
func (g *Gen) getZipf(imax uint64) *rand.Zipf {
|
||||
if nz, ok := g.zipfs[imax]; !ok {
|
||||
nz = rand.NewZipf(g.R, 1.1, 10, imax)
|
||||
g.zipfs[imax] = nz
|
||||
return nz
|
||||
} else {
|
||||
return nz
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Gen) getZipfParams(imax uint64, s, v float64) *rand.Zipf {
|
||||
if nz, ok := g.zipfs[imax]; !ok {
|
||||
nz = rand.NewZipf(g.R, s, v, imax)
|
||||
g.zipfs[imax] = nz
|
||||
return nz
|
||||
} else {
|
||||
return nz
|
||||
}
|
||||
}
|
||||
87
idk/datagen/gen/gen_test.go
Normal file
87
idk/datagen/gen/gen_test.go
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
package gen_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
)
|
||||
|
||||
func TestGen(t *testing.T) {
|
||||
g := gen.New()
|
||||
|
||||
byteSlice := make([]byte, 6)
|
||||
|
||||
// Test AlphaUpper
|
||||
g.AlphaUpper(byteSlice)
|
||||
if len(byteSlice) != 6 {
|
||||
t.Errorf("AlphaUpper modified byte slice length: %s", byteSlice)
|
||||
}
|
||||
for i, b := range byteSlice {
|
||||
if b < 65 || b > 90 {
|
||||
t.Errorf("non uppercase char at position %d: %d", i, b)
|
||||
}
|
||||
}
|
||||
|
||||
// Test AlphaLower
|
||||
g.AlphaLower(byteSlice)
|
||||
if len(byteSlice) != 6 {
|
||||
t.Errorf("AlphaLower modified byte slice length: %s", byteSlice)
|
||||
}
|
||||
for i, b := range byteSlice {
|
||||
if b < 97 || b > 122 {
|
||||
t.Errorf("non lowercase char at position %d: %d", i, b)
|
||||
}
|
||||
}
|
||||
|
||||
// Test StringSliceFromList
|
||||
var input []string
|
||||
list := []string{"a", "b", "c", "d", "e", "f", "g"}
|
||||
|
||||
minLen, maxLen := 8, -1
|
||||
for i := 0; i < 100; i++ {
|
||||
randList := g.StringSliceFromList(input, list)
|
||||
if len(randList) < minLen {
|
||||
minLen = len(randList)
|
||||
}
|
||||
if len(randList) > maxLen {
|
||||
maxLen = len(randList)
|
||||
}
|
||||
for j, item := range randList {
|
||||
if len(item) != 1 || item[0] < 97 || item[0] > 103 {
|
||||
t.Errorf("Unexpected list item at %d: %s. List: %v", j, item, randList)
|
||||
}
|
||||
}
|
||||
}
|
||||
if minLen != 0 {
|
||||
t.Errorf("Unexpected minimum length of %d", minLen)
|
||||
}
|
||||
if maxLen != 7 {
|
||||
t.Errorf("Unexpected maximum length of %d", maxLen)
|
||||
}
|
||||
|
||||
// TestStringFromListWeighted
|
||||
seen := make(map[string]int)
|
||||
for i := 0; i < 100; i++ {
|
||||
s := g.StringFromListWeighted([]string{"blah", "bleh", "blue"})
|
||||
seen[s]++
|
||||
if s != "blah" && s != "bleh" && s != "blue" {
|
||||
t.Errorf("Unexpected string %s", s)
|
||||
}
|
||||
}
|
||||
if len(seen) != 3 {
|
||||
t.Errorf("Should have seen 3 different strings but saw: %v", seen)
|
||||
}
|
||||
|
||||
seen = make(map[string]int)
|
||||
for i := 0; i < 100; i++ {
|
||||
s := g.StringFromListWeighted([]string{"blah", "bleh", "blue", "bler"})
|
||||
seen[s]++
|
||||
if s != "blah" && s != "bleh" && s != "blue" && s != "bler" {
|
||||
t.Errorf("Unexpected string %s", s)
|
||||
}
|
||||
}
|
||||
if len(seen) != 4 {
|
||||
t.Errorf("Should have seen 4 different strings but saw: %v", seen)
|
||||
}
|
||||
|
||||
}
|
||||
114
idk/datagen/hobbies.data.go
Normal file
114
idk/datagen/hobbies.data.go
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
package datagen
|
||||
|
||||
var hobbies = []string{
|
||||
"Lego building", "Watching movies", "Watch making", "Slacklining", "BMX", "Cricket",
|
||||
"Sketching", "Satellite watching", "Volunteering", "Radio-controlled model playing",
|
||||
"Stone collecting", "Picnicking", "Hydroponics", "Karate", "Roller skating", "Skateboarding",
|
||||
"Element collecting", "Weaving", "Beach volleyball", "Archery", "Livestreaming", "Stone skipping",
|
||||
"Trapshooting", "Filmmaking", "Diorama", "Makeup", "Rugby league football", "Community activism",
|
||||
"Field hockey", "Backpacking", "Slot car", "Insect collecting", "VR Gaming", "Video making",
|
||||
"Bowling", "Sled dog racing", "Skiing", "Web design", "Sand art", "Public speaking",
|
||||
"Movie memorabilia collecting", "Gardening", "Wikipedia editing", "Croquet", "Mathematics",
|
||||
"Rail transport modeling", "Darts", "Judo", "Equestrianism", "Figure Skating", "Scrapbooking",
|
||||
"Airbrushing", "Photography", "Climbing", "Tourism", "Journaling", "Flower growing",
|
||||
"Wood carving", "Fashion design", "Polo", "Slot car racing", "Reading", "Electronic games",
|
||||
"Martial arts", "Bell ringing", " Air sports", "Skipping rope", "Bowling", "Caving",
|
||||
"Leather crafting", "Construction", "Bus riding", "Flag football", "Anime", "Whittling",
|
||||
"Aerospace", "Sun bathing", "Music", "Running", "Diving", "Plastic art", "Stamp collecting",
|
||||
"Gymnastics", "Kabaddi", "Coin collecting", "Video editing", "Stripping", "Cribbage",
|
||||
"Candy making", "Amateur geology", "Motor sports", "Sculpting", "Transit map collecting",
|
||||
"Refinishing", "Surfing", "Swimming", "Skateboarding", "Knowledge/word games", "Tether car",
|
||||
"Poi", "Manga", " Action figure", "Teaching", "Blacksmithing", "Fingerpainting", "Audiophile",
|
||||
"Spreadsheets", "Scouting", "Frisbee", "Metal detecting", "Book collecting", "Radio-controlled model playing",
|
||||
"Films", "Karaoke", "Wargaming", "Biology", "DJing", "Axe throwing", "Volleyball",
|
||||
"Life Science", "Fossil hunting", "Beachcombing", "Sudoku", "Cross-stitch", "Ephemera collecting",
|
||||
"Puzzles", "Hiking/backpacking", "Digital hoarding", "Horseshoes", "Amateur astronomy",
|
||||
"Book discussion clubs", "Model building", "Ceramics", "Telling jokes", "Gardening",
|
||||
"Renaissance fair", "Record collecting", "Collecting", "Taxidermy", "Flying", "Zumba",
|
||||
" Archaeology", "Quidditch", "Playing musical instruments", "Tapestry", "Perfume",
|
||||
"Philately", "Business", "Microbiology", "Rafting", "Postcrossing", "Whisky", "Botany",
|
||||
"Badminton", "Chatting", "Board sports", "Groundhopping", "Inventing", "Paragliding",
|
||||
"Shooting sport", "Esports", "Sport stacking", "Proverbs", "Marching band", "Feng shui decorating",
|
||||
"Car tuning", "Sociology", "Writing music", "Robot combat", "Parkour", "Shogi", "Weightlifting",
|
||||
"Fashion", "Safari", "Motorcycling", "Pool", "Meteorology", "Auto audiophilia", "Mushroom hunting/mycology",
|
||||
"Radio-controlled model playing", "Miniature art", "Video game developing", "Medical science",
|
||||
"Herp keeping", "Shoemaking", "Gongfu tea", "Dowsing", "Microscopy", "Welding", "Woodworking",
|
||||
"Clothesmaking", "Fingerprint collecting", "Crossword puzzles", "Breadmaking", "Ice hockey",
|
||||
"Dolls", "Curling", "Sailing", "Mazes (indoor/outdoor)", "Fishkeeping", "Ticket collecting",
|
||||
"Flower arranging", "Nail art", "Couponing", "Skimboarding", "Fishing", "Figure skating",
|
||||
"Herping", "Surfing", "Go", "Vintage clothing", "Shortwave listening", "Water sports",
|
||||
"Darts", "Bonsai", "Lomography", "Crocheting", "Meditation", "Cornhole", "Railway journeys",
|
||||
"Cardistry", "Book restoration", "Graffiti", "Decorating", "Yo-yoing", "Speedcubing",
|
||||
"Lotology (lottery ticket collecting)", "Houseplant care", "Cryptography", "Quilling",
|
||||
"Powerlifting", "Cheesemaking", "Table tennis", "Public transport riding", "Pet adoption & fostering",
|
||||
"Magnet fishing", "Hooping", "Bridge", "Rubik's Cube", "Beekeeping", "Digital arts",
|
||||
"Foreign language learning", "Race walking", "Fusilately (phonecard collecting)",
|
||||
"Fishfarming", "Jigsaw puzzles", "Reviewing Gadgets", "Entrepreneurship", "Pickleball",
|
||||
"Wine tasting", "Footbag", "Astronomy", "Stuffed toy collecting", "Roller derby",
|
||||
"Astrology", "Furniture building", "Lapidary", "Iceboat racing", "High-power rocketry",
|
||||
"Reiki", "Baking", "Automobilism", "Witchcraft", "Walking", "Aerial silk", "Gongoozling",
|
||||
"Learning", "Cartophily (card collecting)", "Paintball", "Genealogy", "Do it yourself",
|
||||
"Volleyball", "Science and technology studies", "Horsemanship", "Swimming", "Needlepoint",
|
||||
"Fishkeeping", "Vintage cars", "Basketball", "Qigong", "Video game collecting", "Writing",
|
||||
"Vacation", "Nordic skating", "Powerboat racing", "Baseball", "Candle making", "Whale watching",
|
||||
"Knot tying", "Ice skating", "Debate", "Checkers (draughts)", "Board/tabletop games",
|
||||
"Model engineering", "VR Gaming", "Palmistry", "Air hockey", "Pole dancing", "Modeling",
|
||||
"Puppetry", "Memory training", "Sculling or rowing", "Seashell collecting", "Poetry",
|
||||
"Role-playing games", "Flying model planes", "Tennis polo", "Gymnastics", "Metalworking",
|
||||
"Scutelliphily", "Eating", "Pet sitting", "Fruit picking", "Farming", "Survivalism",
|
||||
"Fly tying", "Wax sealing", "Sea glass collecting", "Antiquing", "Metal detecting",
|
||||
"Guerrilla gardening", "Dance", "Birdwatching", "Skiing", "Jujitsu", "Hiking", "Model aircraft",
|
||||
"Model United Nations", "Jukskei", "Leaves", "Drama", "Lacrosse", "LARPing", "Home improvement",
|
||||
"Skydiving", "Snowmobiling", "Meteorology", "Fantasy sports", "Blogging", "Hobby horsing",
|
||||
"Knife throwing", "English", "Soapmaking", "Talking", "Lace making", "Driving", "Engraving",
|
||||
"Kung fu", "Laser tag", "Composting", "Sledding", "Croquet", "Railway studies", "Magic",
|
||||
"Kite flying", "Acting", "Juggling", "Travel", "Glassblowing", "Baton twirling", "Boxing",
|
||||
"Kart racing", "Comic book collecting", "Meditation", "Mineral collecting", "Dancing",
|
||||
"Antiquities", "Ultimate frisbee", "Planning", "Pole dancing", "Snorkeling", "Zoo visiting",
|
||||
"Animation", "Rock painting", "Exhibition drill", "Stamp collecting", "People-watching",
|
||||
"Knife collecting", "Herbalism", "Knitting", "Karting", "Tennis", "Drink mixing",
|
||||
"Kombucha brewing", "Chemistry", "Badminton", "Lock picking", "Letterboxing", "Storm chasing",
|
||||
"Sports memorabilia", "Tai chi", "Calligraphy", "Weight training", "Pin (lapel)",
|
||||
"Coffee roasting", "Unicycling", "Ghost hunting", "Archery", "Museum visiting", "Card games",
|
||||
"Dog sport", "Herping", "Netball", "Video gaming", "Trade fair visiting", "Baseball",
|
||||
"plush collecting", "Car fixing & building", "Tatebanko", "BASE jumping", "Gold prospecting",
|
||||
"Animal fancy", "Jogging", "Gunsmithing", "Shooting", "Long-distance running", "Quizzes",
|
||||
"Canoeing", "Aquascaping", "Practical jokes", "Tattooing", "Social studies", "Vehicle restoration",
|
||||
"Cheerleading", "Proofreading and editing", "Fishing", "Squash", "Tarot", "Sewing",
|
||||
"Birdwatching", "Cycling", "Button collecting", "Animation", "Art", "Giving advice",
|
||||
"Handball", "Die-cast toy", "Jewelry making", "Deltiology (postcard collecting)",
|
||||
"Brazilian jiu-jitsu", "Coloring", "Podcast hosting", "Couch surfing", "Reading",
|
||||
"Compact discs", "Bullet journaling", "Hunting", "Australian rules football", "Origami",
|
||||
"Tea bag collecting", "Webtooning", "Longboarding", "Auto detailing", "Hacking", "Kendama",
|
||||
"Photography", "Pilates", "Snowboarding", "Pressed flower craft", "Conlanging", "Beatboxing",
|
||||
"Amateur radio", "Freestyle football", "Mountaineering", "Rock tumbling", "Yoga",
|
||||
"Bus spotting", "Tour skating", "Rock balancing", "Camping", "Sculling or rowing",
|
||||
"Performance", "Djembe", "Entertaining", "Chess", "Cleaning", "Electronics", "Vinyl Records",
|
||||
"Beauty pageants", "Auto racing", "Climbing", "Road biking", "Gingerbread house making",
|
||||
"Distro Hopping", "Geocaching", "Snowshoeing", "Creative writing", "Taekwondo", "Radio-controlled car racing",
|
||||
"Worldbuilding", "Car riding", "Stand-up comedy", "Flying disc", "Dog walking", "Phillumeny",
|
||||
"Foraging", "Singing", "Barbershop Music", "Confectionery", "Amusement park visiting",
|
||||
"Inline skating", "Knife making", "History", "Breakdancing", "Experimenting", "Color guard",
|
||||
"Painting", "Soccer", "Backgammon", "City trip", "Marbles", "Renovating", "Speed skating",
|
||||
"Handball", "Gaming", "Triathlon", "Mountain biking", "Machining", "Art collecting",
|
||||
"Baton twirling", "Horseback riding", "Benchmarking", "Philately", "Tourism", "Wrestling",
|
||||
"Disc golf", "Flower collecting and pressing", "Fitness", "Acroyoga", "Beer tasting",
|
||||
"Video gaming", "Lacrosse", "Bodybuilding", "Thrifting", "Topiary", "3D printing",
|
||||
"Crystals", "Orienteering", "Noodling", "Geocaching", "Orienteering", "Winemaking",
|
||||
"Watching documentaries", "Pet", "Drawing", "Photography", "Airsoft", "Homebrewing",
|
||||
"Aircraft spotting", "Mini Golf", "Storytelling", "Pickleball", "Shuffleboard", "Cooking",
|
||||
"Rock climbing", "Vegetable farming", "Radio-controlled model playing", "Billiards",
|
||||
"Association football", "Embroidery", "Waxing", "Physics", "Hobby tunneling", "Scuba diving",
|
||||
"Kayaking", "Videography", "Tennis", "Slot car", "Table tennis", "Golfing", "Dog training",
|
||||
"Craft", "Mahjong", "Cycling", "Thru-hiking", "Fencing", "Airsoft", "Humor", "Mycology",
|
||||
"Rail transport modelling", "Sports science", "Table football", "Trainspotting", "Minimalism",
|
||||
"Urban exploration", "Macrame", "Computer programming", "Horseback riding", "Cue sports",
|
||||
"Magic", "Pyrography", "Ice skating", "Upcycling", "Shoes", "Power Nap", "Pen Spinning",
|
||||
"Jumping rope", "Astronomy", "Pottery", "Martial arts", "Butterfly watching", "Hula hooping",
|
||||
"Water polo", "Geography", "Chess", "Rugby", "Cosplaying", "Racquetball", "Shopping",
|
||||
"Graphic design", "Binge-watching", "Kitesurfing", "Research", "Model racing", "Listening to podcasts",
|
||||
"Radio-controlled model collecting", "Research", "Rapping", "Poker", "Rappelling",
|
||||
"Watching television", "Listening to music", "Mechanics", "Philosophy", "Recipe creation",
|
||||
"Quilting", "Fossicking", "Social media", "Word searches", "Massaging", "Dominoes",
|
||||
"Longboarding", "Scuba Diving", "Dining", "Hardware", "Communication", "Ant-keeping",
|
||||
"Canyoning", "Dandyism", "Psychology", "Softball", "Table tennis playing"}
|
||||
148
idk/datagen/hughes.go
Normal file
148
idk/datagen/hughes.go
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
)
|
||||
|
||||
// Ensure Hughes implements interface.
|
||||
var _ Sourcer = (*Hughes)(nil)
|
||||
|
||||
// Implements Hughes as a Sourcer.
|
||||
type Hughes struct{}
|
||||
|
||||
// NewHughes returns a new instance of Hughes.
|
||||
func NewHughes(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Hughes{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
// NOTE: avro Name fields can only contain certain
|
||||
// characters.
|
||||
// See: https://avro.apache.org/docs/current/spec.html#names
|
||||
func (e *Hughes) Source(cfg SourceConfig) idk.Source {
|
||||
epoch := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
layout := "2006-01-02 15:00"
|
||||
src := &HughesSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
rand: rand.New(rand.NewSource(int64(cfg.startFrom))),
|
||||
epoch: epoch,
|
||||
layout: layout,
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, //0
|
||||
idk.StringField{NameVal: "sid"}, //1
|
||||
idk.DateIntField{ //2
|
||||
NameVal: "time_stamp",
|
||||
Epoch: epoch,
|
||||
Unit: idk.Hour,
|
||||
Layout: layout,
|
||||
},
|
||||
idk.DecimalField{NameVal: "metered_wan_b_mb_usage", Scale: 2}, //3
|
||||
idk.DateIntField{ //4
|
||||
NameVal: "asap_billing_cycle_gmt_start_date_time",
|
||||
Epoch: epoch,
|
||||
Unit: idk.Hour,
|
||||
Layout: layout,
|
||||
},
|
||||
idk.DecimalField{NameVal: "metered_vsat_wan_a_mb_usage", Scale: 2}, //5
|
||||
},
|
||||
}
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (e *Hughes) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (e *Hughes) DefaultEndAt() uint64 {
|
||||
return 6000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (e *Hughes) Info() string {
|
||||
return "Generates data representative of Baker Hughes' oil field service operations."
|
||||
}
|
||||
|
||||
// Ensure HughesSource implements interface.
|
||||
var _ idk.Source = (*HughesSource)(nil)
|
||||
|
||||
// HughesSource is an instance of a source generated
|
||||
// by the Sourcer implementation Hughes.
|
||||
type HughesSource struct {
|
||||
cur uint64
|
||||
endAt uint64
|
||||
epoch time.Time
|
||||
layout string
|
||||
rand *rand.Rand
|
||||
bDate int
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
// Record implements idk.Source.
|
||||
func (e *HughesSource) Record() (idk.Record, error) {
|
||||
if e.cur >= e.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
e.record[0] = e.cur
|
||||
if e.cur%6000 == 0 { // generates 6000 hours (January 1st to September 6th (23:00)) of data for each new unique SID
|
||||
e.record[1] = e.SIDHughes(24)
|
||||
e.bDate = e.rand.Intn(27) + 1
|
||||
e.record[4] = e.epoch.AddDate(0, -1, e.bDate).Format(e.layout)
|
||||
}
|
||||
e.record[2] = e.epoch.Add(time.Duration(e.cur%6000) * time.Hour).Format(e.layout)
|
||||
e.record[3] = e.floatRand(10000, 0)
|
||||
currDate := e.epoch.Add(time.Duration(e.cur%6000) * time.Hour)
|
||||
if currDate.Day() == e.bDate && currDate.Hour() == 0 {
|
||||
e.record[4] = currDate.Format(e.layout)
|
||||
}
|
||||
e.record[5] = e.floatRand(10000, 0)
|
||||
e.cur++
|
||||
return e.record, nil
|
||||
}
|
||||
|
||||
// Schema implements idk.Source.
|
||||
func (e *HughesSource) Schema() []idk.Field {
|
||||
return e.schema
|
||||
}
|
||||
|
||||
//returns a random float between a range
|
||||
func (e *HughesSource) floatRand(max float64, min float64) float64 {
|
||||
return min + e.rand.Float64()*(max-min)
|
||||
}
|
||||
|
||||
const charsetHughes = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
|
||||
// returns a string with random characters from a predefined charset of a specified length
|
||||
func (e *HughesSource) StringWithCharset(length int, charset string) string {
|
||||
b := make([]byte, length)
|
||||
for i := range b {
|
||||
b[i] = charsetHughes[e.rand.Intn(len(charsetHughes))]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
//returns a string with random English alphabets of the specified length
|
||||
func (e *HughesSource) SIDHughes(length int) string {
|
||||
return e.StringWithCharset(length, charsetHughes)
|
||||
}
|
||||
|
||||
func (e *HughesSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
131
idk/datagen/item.go
Normal file
131
idk/datagen/item.go
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Item implements interface.
|
||||
var _ Sourcer = (*Item)(nil)
|
||||
|
||||
// Item implements Sourcer.
|
||||
type Item struct{}
|
||||
|
||||
// NewItem returns a new instance of Item.
|
||||
func NewItem(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Item{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (i *Item) Source(cfg SourceConfig) idk.Source {
|
||||
min := int64(0)
|
||||
src := &ItemSource{
|
||||
rand: rand.New(rand.NewSource(19)),
|
||||
startFrom: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"},
|
||||
idk.IntField{NameVal: "cust_id"},
|
||||
idk.DecimalField{NameVal: "cost", Scale: 2},
|
||||
idk.IntField{NameVal: "ship_date", Min: &min},
|
||||
idk.StringField{NameVal: "pli"},
|
||||
idk.IDField{NameVal: "prod"},
|
||||
},
|
||||
}
|
||||
src.pliZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(plis))-1)
|
||||
src.prodZipf = rand.NewZipf(src.rand, 1.01, 4, 3200)
|
||||
src.custZipf = rand.NewZipf(src.rand, 1.01, 4, 100000000)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (i *Item) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (i *Item) DefaultEndAt() uint64 {
|
||||
return 1543503872
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (i *Item) Info() string {
|
||||
return "TODO"
|
||||
}
|
||||
|
||||
// Ensure ItemSource implements interface.
|
||||
var _ idk.Source = (*ItemSource)(nil)
|
||||
|
||||
type ItemSource struct {
|
||||
Log logger.Logger
|
||||
|
||||
startFrom uint64
|
||||
endAt uint64
|
||||
|
||||
rand *rand.Rand
|
||||
pliZipf *rand.Zipf
|
||||
prodZipf *rand.Zipf
|
||||
custZipf *rand.Zipf
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
type ItemSourceOption func(s *ItemSource) error
|
||||
|
||||
func OptItemStartFrom(start uint64) ItemSourceOption {
|
||||
return func(s *ItemSource) error {
|
||||
s.startFrom = start
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func OptItemEndAt(end uint64) ItemSourceOption {
|
||||
return func(s *ItemSource) error {
|
||||
s.endAt = end
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
var plis = []string{"laptop", "desktop", "server", "rack", "phone", "fan", "enclosure", "switch", "monitor", "pli1", "pli2", "pli3", "pli4", "pli5", "pli6", "pli7", "pli8", "pli9", "pli10", "pli11", "pli12", "pli13", "pli14", "pli15", "pli16", "pli17", "pli18", "pli19", "pli20", "pli21", "pli22", "pli23", "pli24", "pli25", "pli26", "pli27", "pli28", "pli29", "pli30", "pli31", "pli32", "pli33", "pli34", "pli35", "pli36", "pli37", "pli38", "pli39", "pli40", "pli41", "pli42"}
|
||||
|
||||
func (s *ItemSource) Record() (idk.Record, error) {
|
||||
if s.startFrom >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.startFrom
|
||||
s.startFrom++
|
||||
s.record[1] = int64(s.custZipf.Uint64()) // cust_id
|
||||
s.record[2] = s.rand.Float64() * 10000 // max $10k // cost
|
||||
s.record[3] = int64(time.Duration(s.rand.Intn(int(currentDur))) / (time.Hour * 24)) // random number of days since start ship_date
|
||||
s.record[4] = plis[s.pliZipf.Uint64()]
|
||||
s.record[5] = s.prodZipf.Uint64()
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *ItemSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *ItemSource) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*ItemSource)(nil)
|
||||
|
||||
func (s *ItemSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
163
idk/datagen/kitchen-sink-keyed.go
Normal file
163
idk/datagen/kitchen-sink-keyed.go
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
)
|
||||
|
||||
// Ensure KitchenSinkKeyed implements interface.
|
||||
var _ Sourcer = (*KitchenSinkKeyed)(nil)
|
||||
|
||||
// KitchenSinkKeyed implements Sourcer.
|
||||
type KitchenSinkKeyed struct {
|
||||
schema []idk.Field
|
||||
}
|
||||
|
||||
// NewKitchenSinkKeyed returns a new instance of KitchenSinkKeyed.
|
||||
func NewKitchenSinkKeyed(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &KitchenSinkKeyed{
|
||||
schema: []idk.Field{
|
||||
idk.StringField{NameVal: "pk"}, // 0
|
||||
idk.StringArrayField{NameVal: "set", CacheConfig: &idk.CacheConfig{CacheType: "lru", CacheSize: 1}}, // 1
|
||||
idk.IntField{NameVal: "int", Min: intptr(-9223372036854775807), Max: intptr(9223372036854775807)}, // 2
|
||||
idk.BoolField{NameVal: "bool"}, // 3
|
||||
idk.StringField{NameVal: "time", Quantum: "YMD"}, // 4
|
||||
idk.StringField{NameVal: "mutex", Mutex: true, CacheConfig: &idk.CacheConfig{CacheType: "ranked", CacheSize: 500}}, // 5
|
||||
idk.StringField{NameVal: "string"}, // 6
|
||||
idk.DecimalField{NameVal: "decimal", Scale: 2}, // 7
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (k *KitchenSinkKeyed) Source(cfg SourceConfig) idk.Source {
|
||||
src := &KitchenSinkKeyedSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
schema: k.schema,
|
||||
}
|
||||
|
||||
src.g = gen.New(gen.OptGenSeed(cfg.seed))
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[0] = make([]byte, sha256.Size*(8/4))
|
||||
src.record[6] = make([]byte, 12)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (k *KitchenSinkKeyed) PrimaryKeyFields() []string {
|
||||
return []string{"pk"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (k *KitchenSinkKeyed) DefaultEndAt() uint64 {
|
||||
return 20000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (k *KitchenSinkKeyed) Info() string {
|
||||
return "Generates data to test everything but the kitchen sink. With Keys."
|
||||
}
|
||||
|
||||
// Ensure KitchenSinkKeyedSource implements interface.
|
||||
var _ idk.Source = (*KitchenSinkKeyedSource)(nil)
|
||||
|
||||
type KitchenSinkKeyedSource struct {
|
||||
g *gen.Gen
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
func NewKitchenSinkKeyedSource(start, end uint64) *KitchenSinkKeyedSource {
|
||||
src := &KitchenSinkKeyedSource{
|
||||
cur: start,
|
||||
endAt: end,
|
||||
|
||||
schema: []idk.Field{
|
||||
idk.StringField{NameVal: "pk"}, // 0
|
||||
idk.StringArrayField{NameVal: "set", CacheConfig: &idk.CacheConfig{CacheType: "lru", CacheSize: 1}}, // 1
|
||||
idk.IntField{NameVal: "int", Min: intptr(-9223372036854775807), Max: intptr(9223372036854775807)}, // 2
|
||||
idk.BoolField{NameVal: "bool"}, // 3
|
||||
idk.StringField{NameVal: "time", Quantum: "YMD"}, // 4
|
||||
idk.StringField{NameVal: "mutex", Mutex: true, CacheConfig: &idk.CacheConfig{CacheType: "ranked", CacheSize: 500}}, // 5
|
||||
idk.StringField{NameVal: "string"}, // 6
|
||||
idk.DecimalField{NameVal: "decimal", Scale: 2}, // 7
|
||||
},
|
||||
}
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[0] = make([]byte, sha256.Size*(8/4))
|
||||
src.record[6] = make([]byte, 12)
|
||||
|
||||
return src
|
||||
|
||||
}
|
||||
|
||||
func (k *KitchenSinkKeyedSource) Record() (idk.Record, error) {
|
||||
if k.cur >= k.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
// Generate a unique string key.
|
||||
var nbytes [64 / 8]byte
|
||||
binary.LittleEndian.PutUint64(nbytes[:], k.cur)
|
||||
hash := sha256.Sum256(nbytes[:])
|
||||
hex.Encode(k.record[0].([]byte), hash[:])
|
||||
k.cur++
|
||||
|
||||
// Generate a random set field.
|
||||
set := k.g.Set(100, 1000, 10)
|
||||
vals := make([]string, 0, len(set))
|
||||
for v := range set {
|
||||
vals = append(vals, "v"+strconv.FormatUint(v, 10))
|
||||
}
|
||||
sort.Strings(vals)
|
||||
k.record[1] = vals
|
||||
|
||||
// Generate a random int field.
|
||||
k.record[2] = ((2 * k.g.R.Int63n(2)) - 1) * k.g.R.Int63n(9223372036854775807)
|
||||
|
||||
// Generate a random bool field.
|
||||
k.record[3] = k.g.R.Intn(2) == 0
|
||||
|
||||
// Generate a random time field.
|
||||
k.record[4] = startTime.Add(time.Duration(k.g.R.Int63n(int64(timeSpan))))
|
||||
|
||||
// Generate a random mutex value.
|
||||
k.record[5] = strconv.Itoa(k.g.R.Intn(20))
|
||||
|
||||
// Generate a random string field.
|
||||
k.g.AlphaUpper(k.record[6].([]byte))
|
||||
|
||||
// Generate a random floating point value.
|
||||
k.record[7] = float64(k.g.R.Int63()) / 100
|
||||
|
||||
return k.record, nil
|
||||
}
|
||||
|
||||
func (k *KitchenSinkKeyedSource) Schema() []idk.Field {
|
||||
return k.schema
|
||||
}
|
||||
|
||||
func (k *KitchenSinkKeyedSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
139
idk/datagen/kitchen-sink.go
Normal file
139
idk/datagen/kitchen-sink.go
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
"github.com/molecula/featurebase/v3/pql"
|
||||
)
|
||||
|
||||
// Ensure KitchenSink implements interface.
|
||||
var _ Sourcer = (*KitchenSink)(nil)
|
||||
|
||||
// KitchenSink implements Sourcer.
|
||||
type KitchenSink struct {
|
||||
schema []idk.Field
|
||||
}
|
||||
|
||||
// NewKitchenSink returns a new instance of KitchenSink.
|
||||
func NewKitchenSink(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &KitchenSink{
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringArrayField{NameVal: "set", CacheConfig: &idk.CacheConfig{CacheType: "lru", CacheSize: 1}}, // 1
|
||||
idk.IntField{NameVal: "int", Min: intptr(-9223372036854775807), Max: intptr(9223372036854775807)}, // 2
|
||||
idk.BoolField{NameVal: "bool"}, // 3
|
||||
idk.StringField{NameVal: "time", Quantum: "YMD"}, // 4
|
||||
idk.StringField{NameVal: "mutex", Mutex: true, CacheConfig: &idk.CacheConfig{CacheType: "ranked", CacheSize: 500}}, // 5
|
||||
idk.StringField{NameVal: "string"}, // 6
|
||||
idk.DecimalField{NameVal: "decimal", Scale: 2}, // 7
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (k *KitchenSink) Source(cfg SourceConfig) idk.Source {
|
||||
src := &KitchenSinkSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
schema: k.schema,
|
||||
}
|
||||
|
||||
src.g = gen.New(gen.OptGenSeed(cfg.seed))
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[0] = uint64(0)
|
||||
src.record[6] = make([]byte, 12)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (k *KitchenSink) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (k *KitchenSink) DefaultEndAt() uint64 {
|
||||
return 20000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (k *KitchenSink) Info() string {
|
||||
return "Generates data to test everything but the kitchen sink."
|
||||
}
|
||||
|
||||
// Ensure KitchenSinkSource implements interface.
|
||||
var _ idk.Source = (*KitchenSinkSource)(nil)
|
||||
|
||||
// KitchenSinkSource is a data generator which generates
|
||||
// data for all Pilosa field types.
|
||||
type KitchenSinkSource struct {
|
||||
g *gen.Gen
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
var endTime = time.Date(2020, time.May, 4, 12, 2, 28, 0, time.UTC)
|
||||
var startTime = endTime.Add(-5 * 365 * 24 * time.Hour)
|
||||
var timeSpan = endTime.Sub(startTime)
|
||||
|
||||
func (k *KitchenSinkSource) Record() (idk.Record, error) {
|
||||
if k.cur >= k.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
// Increment the ID.
|
||||
k.record[0] = k.cur
|
||||
k.cur++
|
||||
|
||||
// Generate a random set field.
|
||||
set := k.g.Set(100, 1000, 10)
|
||||
vals := make([]string, 0, len(set))
|
||||
for v := range set {
|
||||
vals = append(vals, "v"+strconv.FormatUint(v, 10))
|
||||
}
|
||||
sort.Strings(vals)
|
||||
k.record[1] = vals
|
||||
|
||||
// Generate a random int field.
|
||||
k.record[2] = ((2 * k.g.R.Int63n(2)) - 1) * k.g.R.Int63n(9223372036854775807)
|
||||
|
||||
// Generate a random bool field.
|
||||
k.record[3] = k.g.R.Intn(2) == 0
|
||||
|
||||
// Generate a random time field.
|
||||
k.record[4] = startTime.Add(time.Duration(k.g.R.Int63n(int64(timeSpan))))
|
||||
|
||||
// Generate a random mutex value.
|
||||
k.record[5] = strconv.Itoa(k.g.R.Intn(20))
|
||||
|
||||
// Generate a random string field.
|
||||
k.g.AlphaUpper(k.record[6].([]byte))
|
||||
|
||||
// Generate a random floating point value.
|
||||
k.record[7] = pql.NewDecimal(k.g.R.Int63(), 2)
|
||||
|
||||
return k.record, nil
|
||||
}
|
||||
|
||||
func (k *KitchenSinkSource) Schema() []idk.Field {
|
||||
return k.schema
|
||||
}
|
||||
|
||||
func (k *KitchenSinkSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
104535
idk/datagen/locations.data.go
Normal file
104535
idk/datagen/locations.data.go
Normal file
File diff suppressed because it is too large
Load diff
721
idk/datagen/merck.go
Normal file
721
idk/datagen/merck.go
Normal file
|
|
@ -0,0 +1,721 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
)
|
||||
|
||||
// Ensure Merck implements interface.
|
||||
var _ Sourcer = (*Merck)(nil)
|
||||
|
||||
// Merck implements Sourcer, and returns a very basic
|
||||
// data set. It can be used as an Merck for writing
|
||||
// additional custom Sourcers.
|
||||
type Merck struct{}
|
||||
|
||||
// NewMerck returns a new instance of Merck.
|
||||
func NewMerck(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Merck{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (e *Merck) Source(cfg SourceConfig) idk.Source {
|
||||
src := &MerckSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
rand: rand.New(rand.NewSource(19)),
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, //0
|
||||
idk.StringField{NameVal: "image_location"}, //1
|
||||
idk.StringField{NameVal: "image_tag"}, //2
|
||||
idk.StringField{NameVal: "algorithm_name"}, //3
|
||||
idk.StringField{NameVal: "start_time"}, //4
|
||||
idk.StringField{NameVal: "analysis_region"}, //5
|
||||
idk.IntField{NameVal: "total_cells", Min: intptr(1000), Max: intptr(1000000)}, //6
|
||||
idk.IntField{NameVal: "dye_1_positive_cells", Min: intptr(0), Max: intptr(1000000)}, //7
|
||||
idk.IntField{NameVal: "dye_2_positive_cells", Min: intptr(0), Max: intptr(1000000)}, //8
|
||||
idk.IntField{NameVal: "dye_3_positive_cells", Min: intptr(0), Max: intptr(1000000)}, //9
|
||||
idk.IntField{NameVal: "dye_4_positive_cells", Min: intptr(0), Max: intptr(1000000)}, //10
|
||||
idk.IntField{NameVal: "dye_5_positive_cells", Min: intptr(0), Max: intptr(1000000)}, //11
|
||||
idk.IntField{NameVal: "dye_6_positive_cells", Min: intptr(0), Max: intptr(1000000)}, //12
|
||||
idk.IntField{NameVal: "dye_7_positive_cells", Min: intptr(0), Max: intptr(1000000)}, //13
|
||||
idk.IntField{NameVal: "dye_8_positive_cells", Min: intptr(0), Max: intptr(1000000)}, //14
|
||||
idk.IntField{NameVal: "dye_1_positive_nuclei_cells", Min: intptr(0), Max: intptr(1000000)}, //15
|
||||
idk.IntField{NameVal: "dye_2_positive_nuclei_cells", Min: intptr(0), Max: intptr(1000000)}, //16
|
||||
idk.IntField{NameVal: "dye_3_positive_nuclei_cells", Min: intptr(0), Max: intptr(1000000)}, //17
|
||||
idk.IntField{NameVal: "dye_4_positive_nuclei_cells", Min: intptr(0), Max: intptr(1000000)}, //18
|
||||
idk.IntField{NameVal: "dye_5_positive_nuclei_cells", Min: intptr(0), Max: intptr(1000000)}, //19
|
||||
idk.IntField{NameVal: "dye_6_positive_nuclei_cells", Min: intptr(0), Max: intptr(1000000)}, //20
|
||||
idk.IntField{NameVal: "dye_7_positive_nuclei_cells", Min: intptr(0), Max: intptr(1000000)}, //21
|
||||
idk.IntField{NameVal: "dye_8_positive_nuclei_cells", Min: intptr(0), Max: intptr(1000000)}, //22
|
||||
idk.IntField{NameVal: "negative_nuclei_cells", Min: intptr(0), Max: intptr(1000000)}, //23
|
||||
idk.IntField{NameVal: "dye_1_positive_cytoplasm_cells", Min: intptr(0), Max: intptr(1000000)}, //24
|
||||
idk.IntField{NameVal: "dye_2_positive_cytoplasm_cells", Min: intptr(0), Max: intptr(1000000)}, //25
|
||||
idk.IntField{NameVal: "dye_3_positive_cytoplasm_cells", Min: intptr(0), Max: intptr(1000000)}, //26
|
||||
idk.IntField{NameVal: "dye_4_positive_cytoplasm_cells", Min: intptr(0), Max: intptr(1000000)}, //27
|
||||
idk.IntField{NameVal: "dye_5_positive_cytoplasm_cells", Min: intptr(0), Max: intptr(1000000)}, //28
|
||||
idk.IntField{NameVal: "dye_6_positive_cytoplasm_cells", Min: intptr(0), Max: intptr(1000000)}, //29
|
||||
idk.IntField{NameVal: "dye_7_positive_cytoplasm_cells", Min: intptr(0), Max: intptr(1000000)}, //30
|
||||
idk.IntField{NameVal: "dye_8_positive_cytoplasm_cells", Min: intptr(0), Max: intptr(1000000)}, //31
|
||||
idk.IntField{NameVal: "negative_cytoplasm_cells", Min: intptr(0), Max: intptr(1000000)}, //32
|
||||
idk.IntField{NameVal: "dye_1_positive_membrane_cells", Min: intptr(0), Max: intptr(1000000)}, //33
|
||||
idk.IntField{NameVal: "dye_2_positive_membrane_cells", Min: intptr(0), Max: intptr(1000000)}, //34
|
||||
idk.IntField{NameVal: "dye_3_positive_membrane_cells", Min: intptr(0), Max: intptr(1000000)}, //35
|
||||
idk.IntField{NameVal: "dye_4_positive_membrane_cells", Min: intptr(0), Max: intptr(1000000)}, //36
|
||||
idk.IntField{NameVal: "dye_5_positive_membrane_cells", Min: intptr(0), Max: intptr(1000000)}, //37
|
||||
idk.IntField{NameVal: "dye_6_positive_membrane_cells", Min: intptr(0), Max: intptr(1000000)}, //38
|
||||
idk.IntField{NameVal: "dye_7_positive_membrane_cells", Min: intptr(0), Max: intptr(1000000)}, //39
|
||||
idk.IntField{NameVal: "dye_8_positive_membrane_cells", Min: intptr(0), Max: intptr(1000000)}, //40
|
||||
idk.IntField{NameVal: "negative_membrane_cells", Min: intptr(0), Max: intptr(1000000)}, //41
|
||||
idk.DecimalField{NameVal: "dye_1_per_positive_cells", Scale: 6}, //42
|
||||
idk.DecimalField{NameVal: "dye_2_per_positive_cells", Scale: 6}, //43
|
||||
idk.DecimalField{NameVal: "dye_3_per_positive_cells", Scale: 6}, //44
|
||||
idk.DecimalField{NameVal: "dye_4_per_positive_cells", Scale: 6}, //45
|
||||
idk.DecimalField{NameVal: "dye_5_per_positive_cells", Scale: 6}, //46
|
||||
idk.DecimalField{NameVal: "dye_6_per_positive_cells", Scale: 6}, //47
|
||||
idk.DecimalField{NameVal: "dye_7_per_positive_cells", Scale: 6}, //48
|
||||
idk.DecimalField{NameVal: "dye_8_per_positive_cells", Scale: 6}, //49
|
||||
idk.DecimalField{NameVal: "dye_1_per_positive_nuclei_cells", Scale: 6}, //50
|
||||
idk.DecimalField{NameVal: "dye_2_per_positive_nuclei_cells", Scale: 6}, //51
|
||||
idk.DecimalField{NameVal: "dye_3_per_positive_nuclei_cells", Scale: 6}, //52
|
||||
idk.DecimalField{NameVal: "dye_4_per_positive_nuclei_cells", Scale: 6}, //53
|
||||
idk.DecimalField{NameVal: "dye_5_per_positive_nuclei_cells", Scale: 6}, //54
|
||||
idk.DecimalField{NameVal: "dye_6_per_positive_nuclei_cells", Scale: 6}, //55
|
||||
idk.DecimalField{NameVal: "dye_7_per_positive_nuclei_cells", Scale: 6}, //56
|
||||
idk.DecimalField{NameVal: "dye_8_per_positive_nuclei_cells", Scale: 6}, //57
|
||||
idk.DecimalField{NameVal: "dye_1_per_positive_cytoplasm_cells", Scale: 6}, //58
|
||||
idk.DecimalField{NameVal: "dye_2_per_positive_cytoplasm_cells", Scale: 6}, //59
|
||||
idk.DecimalField{NameVal: "dye_3_per_positive_cytoplasm_cells", Scale: 6}, //60
|
||||
idk.DecimalField{NameVal: "dye_4_per_positive_cytoplasm_cells", Scale: 6}, //61
|
||||
idk.DecimalField{NameVal: "dye_5_per_positive_cytoplasm_cells", Scale: 6}, //62
|
||||
idk.DecimalField{NameVal: "dye_6_per_positive_cytoplasm_cells", Scale: 6}, //63
|
||||
idk.DecimalField{NameVal: "dye_7_per_positive_cytoplasm_cells", Scale: 6}, //64
|
||||
idk.DecimalField{NameVal: "dye_8_per_positive_cytoplasm_cells", Scale: 6}, //65
|
||||
idk.DecimalField{NameVal: "dye_1_per_positive_membrane_cells", Scale: 6}, //66
|
||||
idk.DecimalField{NameVal: "dye_2_per_positive_membrane_cells", Scale: 6}, //67
|
||||
idk.DecimalField{NameVal: "dye_3_per_positive_membrane_cells", Scale: 6}, //68
|
||||
idk.DecimalField{NameVal: "dye_4_per_positive_membrane_cells", Scale: 6}, //69
|
||||
idk.DecimalField{NameVal: "dye_5_per_positive_membrane_cells", Scale: 6}, //70
|
||||
idk.DecimalField{NameVal: "dye_6_per_positive_membrane_cells", Scale: 6}, //71
|
||||
idk.DecimalField{NameVal: "dye_7_per_positive_membrane_cells", Scale: 6}, //72
|
||||
idk.DecimalField{NameVal: "dye_8_per_positive_membrane_cells", Scale: 6}, //73
|
||||
idk.DecimalField{NameVal: "dye_1_positive_nuclei_avg_intensity", Scale: 6}, //74
|
||||
idk.DecimalField{NameVal: "dye_1_positive_cytoplasm_avg_intensity", Scale: 6}, //75
|
||||
idk.DecimalField{NameVal: "dye_1_positive_membrane_avg_intensity", Scale: 6}, //76
|
||||
idk.DecimalField{NameVal: "dye_2_positive_nuclei_avg_intensity", Scale: 6}, //77
|
||||
idk.DecimalField{NameVal: "dye_2_positive_cytoplasm_avg_intensity", Scale: 6}, //78
|
||||
idk.DecimalField{NameVal: "dye_2_positive_membrane_avg_intensity", Scale: 6}, //79
|
||||
idk.DecimalField{NameVal: "dye_3_positive_nuclei_avg_intensity", Scale: 6}, //80
|
||||
idk.DecimalField{NameVal: "dye_3_positive_cytoplasm_avg_intensity", Scale: 6}, //81
|
||||
idk.DecimalField{NameVal: "dye_3_positive_membrane_avg_intensity", Scale: 6}, //82
|
||||
idk.DecimalField{NameVal: "dye_4_positive_nuclei_avg_intensity", Scale: 6}, //83
|
||||
idk.DecimalField{NameVal: "dye_4_positive_cytoplasm_avg_intensity", Scale: 6}, //84
|
||||
idk.DecimalField{NameVal: "dye_4_positive_membrane_avg_intensity", Scale: 6}, //85
|
||||
idk.DecimalField{NameVal: "dye_5_positive_nuclei_avg_intensity", Scale: 6}, //86
|
||||
idk.DecimalField{NameVal: "dye_5_positive_cytoplasm_avg_intensity", Scale: 6}, //87
|
||||
idk.DecimalField{NameVal: "dye_5_positive_membrane_avg_intensity", Scale: 6}, //88
|
||||
idk.DecimalField{NameVal: "dye_6_positive_nuclei_avg_intensity", Scale: 6}, //89
|
||||
idk.DecimalField{NameVal: "dye_6_positive_cytoplasm_avg_intensity", Scale: 6}, //90
|
||||
idk.DecimalField{NameVal: "dye_6_positive_membrane_avg_intensity", Scale: 6}, //91
|
||||
idk.DecimalField{NameVal: "dye_7_positive_nuclei_avg_intensity", Scale: 6}, //92
|
||||
idk.DecimalField{NameVal: "dye_7_positive_cytoplasm_avg_intensity", Scale: 6}, //93
|
||||
idk.DecimalField{NameVal: "dye_7_positive_membrane_avg_intensity", Scale: 6}, //94
|
||||
idk.DecimalField{NameVal: "dye_8_positive_nuclei_avg_intensity", Scale: 6}, //95
|
||||
idk.DecimalField{NameVal: "dye_8_positive_cytoplasm_avg_intensity", Scale: 6}, //96
|
||||
idk.DecimalField{NameVal: "dye_8_positive_membrane_avg_intensity", Scale: 6}, //97
|
||||
idk.DecimalField{NameVal: "avg_nucleus_area_um2", Scale: 6}, //98
|
||||
idk.DecimalField{NameVal: "avg_cytoplasm_area_um2", Scale: 6}, //99
|
||||
idk.DecimalField{NameVal: "avg_cell_area_um2", Scale: 6}, //100
|
||||
idk.IntField{NameVal: "area_analyzed_um2"}, //101
|
||||
idk.IntField{NameVal: "image_zoom"}, //102
|
||||
idk.BoolField{NameVal: "classifier"}, //103
|
||||
idk.StringField{NameVal: "class_list"}, //104
|
||||
idk.BoolField{NameVal: "classify_registered"}, //105
|
||||
idk.StringField{NameVal: "classifier_output_type"}, //106
|
||||
idk.StringField{NameVal: "dye_1"}, //107
|
||||
idk.BoolField{NameVal: "dye_1_membrane_segmentation"}, //108
|
||||
idk.DecimalField{NameVal: "dye_1_nucleus_weight", Scale: 3}, //109
|
||||
idk.DecimalField{NameVal: "dye_1_nucleus_positive_threshold", Scale: 3}, //110
|
||||
idk.DecimalField{NameVal: "dye_1_cytoplasm_positive_threshold", Scale: 3}, //111
|
||||
idk.DecimalField{NameVal: "dye_1_membrane_positive_threshold", Scale: 3}, //112
|
||||
idk.StringField{NameVal: "dye_1_nucleus_mask"}, //113
|
||||
idk.StringField{NameVal: "dye_1_cytoplasm_mask"}, //114
|
||||
idk.StringField{NameVal: "dye_1_membrane_mask"}, //115
|
||||
idk.StringField{NameVal: "dye_2"}, //116
|
||||
idk.BoolField{NameVal: "dye_2_membrane_segmentation"}, //117
|
||||
idk.DecimalField{NameVal: "dye_2_nucleus_weight", Scale: 3}, //118
|
||||
idk.DecimalField{NameVal: "dye_2_nucleus_positive_threshold", Scale: 3}, //119
|
||||
idk.DecimalField{NameVal: "dye_2_cytoplasm_positive_threshold", Scale: 3}, //120
|
||||
idk.DecimalField{NameVal: "dye_2_membrane_positive_threshold", Scale: 3}, //121
|
||||
idk.StringField{NameVal: "dye_2_nucleus_mask"}, //122
|
||||
idk.StringField{NameVal: "dye_2_cytoplasm_mask"}, //123
|
||||
idk.StringField{NameVal: "dye_2_membrane_mask"}, //124
|
||||
idk.StringField{NameVal: "dye_3"}, //125
|
||||
idk.BoolField{NameVal: "dye_3_membrane_segmentation"}, //126
|
||||
idk.DecimalField{NameVal: "dye_3_nucleus_weight", Scale: 3}, //127
|
||||
idk.DecimalField{NameVal: "dye_3_nucleus_positive_threshold", Scale: 3}, //128
|
||||
idk.DecimalField{NameVal: "dye_3_cytoplasm_positive_threshold", Scale: 3}, //129
|
||||
idk.DecimalField{NameVal: "dye_3_membrane_positive_threshold", Scale: 3}, //130
|
||||
idk.StringField{NameVal: "dye_3_nucleus_mask"}, //131
|
||||
idk.StringField{NameVal: "dye_3_cytoplasm_mask"}, //132
|
||||
idk.StringField{NameVal: "dye_3_membrane_mask"}, //133
|
||||
idk.StringField{NameVal: "dye_4"}, //134
|
||||
idk.BoolField{NameVal: "dye_4_membrane_segmentation"}, //135
|
||||
idk.DecimalField{NameVal: "dye_4_nucleus_weight", Scale: 3}, //136
|
||||
idk.DecimalField{NameVal: "dye_4_nucleus_positive_threshold", Scale: 3}, //137
|
||||
idk.DecimalField{NameVal: "dye_4_cytoplasm_positive_threshold", Scale: 3}, //138
|
||||
idk.DecimalField{NameVal: "dye_4_membrane_positive_threshold", Scale: 3}, //139
|
||||
idk.StringField{NameVal: "dye_4_nucleus_mask"}, //140
|
||||
idk.StringField{NameVal: "dye_4_cytoplasm_mask"}, //141
|
||||
idk.StringField{NameVal: "dye_4_membrane_mask"}, //142
|
||||
idk.StringField{NameVal: "dye_5"}, //143
|
||||
idk.BoolField{NameVal: "dye_5_membrane_segmentation"}, //144
|
||||
idk.DecimalField{NameVal: "dye_5_nucleus_weight", Scale: 3}, //145
|
||||
idk.DecimalField{NameVal: "dye_5_nucleus_positive_threshold", Scale: 3}, //146
|
||||
idk.DecimalField{NameVal: "dye_5_cytoplasm_positive_threshold", Scale: 3}, //147
|
||||
idk.DecimalField{NameVal: "dye_5_membrane_positive_threshold", Scale: 3}, //148
|
||||
idk.StringField{NameVal: "dye_5_nucleus_mask"}, //149
|
||||
idk.StringField{NameVal: "dye_5_cytoplasm_mask"}, //150
|
||||
idk.StringField{NameVal: "dye_5_membrane_mask"}, //151
|
||||
idk.StringField{NameVal: "dye_6"}, //152
|
||||
idk.BoolField{NameVal: "dye_6_membrane_segmentation"}, //153
|
||||
idk.DecimalField{NameVal: "dye_6_nucleus_weight", Scale: 3}, //154
|
||||
idk.DecimalField{NameVal: "dye_6_nucleus_positive_threshold", Scale: 3}, //155
|
||||
idk.DecimalField{NameVal: "dye_6_cytoplasm_positive_threshold", Scale: 3}, //156
|
||||
idk.DecimalField{NameVal: "dye_6_membrane_positive_threshold", Scale: 3}, //157
|
||||
idk.StringField{NameVal: "dye_6_nucleus_mask"}, //158
|
||||
idk.StringField{NameVal: "dye_6_cytoplasm_mask"}, //159
|
||||
idk.StringField{NameVal: "dye_6_membrane_mask"}, //160
|
||||
idk.StringField{NameVal: "dye_7"}, //161
|
||||
idk.BoolField{NameVal: "dye_7_membrane_segmentation"}, //162
|
||||
idk.DecimalField{NameVal: "dye_7_nucleus_weight", Scale: 3}, //163
|
||||
idk.DecimalField{NameVal: "dye_7_nucleus_positive_threshold", Scale: 3}, //164
|
||||
idk.DecimalField{NameVal: "dye_7_cytoplasm_positive_threshold", Scale: 3}, //165
|
||||
idk.DecimalField{NameVal: "dye_7_membrane_positive_threshold", Scale: 3}, //166
|
||||
idk.StringField{NameVal: "dye_7_nucleus_mask"}, //167
|
||||
idk.StringField{NameVal: "dye_7_cytoplasm_mask"}, //168
|
||||
idk.StringField{NameVal: "dye_7_membrane_mask"}, //169
|
||||
idk.StringField{NameVal: "dye_8"}, //170
|
||||
idk.BoolField{NameVal: "dye_8_membrane_segmentation"}, //171
|
||||
idk.DecimalField{NameVal: "dye_8_nucleus_weight", Scale: 3}, //172
|
||||
idk.DecimalField{NameVal: "dye_8_nucleus_positive_threshold", Scale: 3}, //173
|
||||
idk.DecimalField{NameVal: "dye_8_cytoplasm_positive_threshold", Scale: 3}, //174
|
||||
idk.DecimalField{NameVal: "dye_8_membrane_positive_threshold", Scale: 3}, //175
|
||||
idk.StringField{NameVal: "dye_8_nucleus_mask"}, //176
|
||||
idk.StringField{NameVal: "dye_8_cytoplasm_mask"}, //177
|
||||
idk.StringField{NameVal: "dye_8_membrane_mask"}, //178
|
||||
idk.DecimalField{NameVal: "nuclear_contrast_threshold", Scale: 3}, //179
|
||||
idk.DecimalField{NameVal: "minimum_nuclear_intensity", Scale: 3}, //180
|
||||
idk.DecimalField{NameVal: "nuclear_segmentation_aggressiveness", Scale: 3}, //181
|
||||
idk.DecimalField{NameVal: "fill_nuclear_holes", Scale: 3}, //182
|
||||
idk.DecimalField{NameVal: "minimum_nuclear_size", Scale: 3}, //183
|
||||
idk.DecimalField{NameVal: "maximum_nuclear_size", Scale: 3}, //184
|
||||
idk.DecimalField{NameVal: "minimum_nuclear_roundness", Scale: 3}, //185
|
||||
idk.DecimalField{NameVal: "maximum_cytoplasm_radius", Scale: 3}, //186
|
||||
idk.DecimalField{NameVal: "membrane_segmentation_agressiveness", Scale: 3}, //187
|
||||
idk.DecimalField{NameVal: "minimum_cell_size", Scale: 3}, //188
|
||||
idk.DecimalField{NameVal: "maximum_cell_size", Scale: 3}, //189
|
||||
idk.StringField{NameVal: "output_image"}, //190
|
||||
idk.DecimalField{NameVal: "store_object_cell_data"}, //191
|
||||
idk.DecimalField{NameVal: "nuclear_size_minimum", Scale: 3}, //192
|
||||
idk.DecimalField{NameVal: "nuclear_size_maximum", Scale: 3}, //193
|
||||
idk.DecimalField{NameVal: "cell_size_minimum", Scale: 3}, //194
|
||||
idk.DecimalField{NameVal: "cell_size_maximum", Scale: 3}, //195
|
||||
idk.IDField{NameVal: "subject_id"}, //196
|
||||
idk.StringField{NameVal: "su_species"}, //197
|
||||
idk.StringField{NameVal: "su_sex"}, //198
|
||||
idk.StringField{NameVal: "su_date_of_birth"}, //199
|
||||
idk.IDField{NameVal: "case_id"}, //200
|
||||
idk.IntField{NameVal: "c_age"}, //201
|
||||
idk.IntField{NameVal: "c_weight"}, //202
|
||||
idk.IDArrayField{NameVal: "procedure_id"}, //203
|
||||
idk.StringArrayField{NameVal: "p_name"}, //204
|
||||
idk.StringArrayField{NameVal: "p_description"}, //205
|
||||
idk.IntField{NameVal: "p_ontology_code"}, //206
|
||||
idk.IDField{NameVal: "diagnosis_id"}, //207
|
||||
idk.StringField{NameVal: "dx_comments"}, //208
|
||||
idk.IDField{NameVal: "disease_id"}, //209
|
||||
idk.StringField{NameVal: "dz_disease_name"}, //210
|
||||
idk.StringField{NameVal: "dz_etiology"}, //211
|
||||
idk.StringField{NameVal: "dz_morphology"}, //212
|
||||
idk.IntField{NameVal: "dz_ontology_code"}, //213
|
||||
idk.IDField{NameVal: "specimen_id"}, //214
|
||||
idk.StringField{NameVal: "sp_tissue_type"}, //215
|
||||
idk.IntField{NameVal: "sp_sample_volume"}, //216
|
||||
idk.StringField{NameVal: "sp_sample_location"}, //217
|
||||
},
|
||||
}
|
||||
src.Z = gen.New(gen.OptGenSeed(cfg.seed))
|
||||
src.algZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(alg))-1)
|
||||
src.maskZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(mask))-1)
|
||||
src.analysisregionZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(analysisregion))-1)
|
||||
src.classifierTypeZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(classifierType))-1)
|
||||
src.classListZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(classList))-1)
|
||||
src.outputTypeZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(outputType))-1)
|
||||
src.dyeZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(dye))-1)
|
||||
src.outputImageZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(outputImage))-1)
|
||||
src.speciesZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(species))-1)
|
||||
src.sexZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(sex))-1)
|
||||
src.commentsZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(comments))-1)
|
||||
src.locationTissueZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(locationTissue))-1)
|
||||
src.tissuetypeZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(tissuetype))-1)
|
||||
src.etiologyZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(etiology))-1)
|
||||
src.morphologyZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(morphology))-1)
|
||||
src.procedures_descZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(procedures_desc))-1)
|
||||
src.diseaseZipf = rand.NewZipf(src.rand, 1.01, 4, uint64(len(disease))-1)
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (e *Merck) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (e *Merck) DefaultEndAt() uint64 {
|
||||
return 100
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (e *Merck) Info() string {
|
||||
return "Generates data representative of Merck's pharmaceutical operations."
|
||||
}
|
||||
|
||||
var alg = []string{"NSCLC T", "TC1_28", "NSCLC-LY-21-LA", "NSCLC-LY-22-LA", "NSCLC-LY-23-LA", "NSCLC-LY-24-LA", "NSCLC-LY-25-LA", "NSCLC-LY-37-LA", "NSCLC-LY-38-LA", "NSCLC-LY-39-LA", "NSCLC-LY-40-LA", "NSCLC-LY-41-LA", "NSCLC-LY-42-LA", "NSCLC-LY-43-LA", "NSCLC-LY-44-LA", "NSCLC-LY-45-LA", "NSCLC-LY-46-LA", "NSCLC-LY-47-LA", "NSCLC-LY-48-LA", "NSCLC-LY-59-LA", "NSCLC-LY-60-LA"}
|
||||
|
||||
var mask = []string{"not set", "OR Negative", "OR Positive"}
|
||||
|
||||
var species = []string{"Homo sapiens", "Rattus", "Sus scrofa"}
|
||||
|
||||
var sex = []string{"M", "F", "U"}
|
||||
|
||||
var etiology = []string{"Smoking", "Sun light", "Diet Coke", "Sneezing", "Open Cut"}
|
||||
|
||||
var morphology = []string{"Lungs", "Skin", "Blood", "Liver", "Stomach"}
|
||||
|
||||
var locationTissue = []string{"US", "UK", "CAN", "IND"}
|
||||
|
||||
var disease = []string{"Cancer", "Diabetes", "Thyroid", "Leukemia"}
|
||||
|
||||
var tissuetype = []string{"MUSC", "NERV", "EPIT", "CONN"}
|
||||
|
||||
var procedures_desc = []string{"Cut muscle", "Stich muscle", "Remove skin", "Add oil", "Triple bypass ", "Tape bone", "Remove organ", "Unplug", "Sedate", "Apply bandaid"}
|
||||
|
||||
var comments = []string{"aliquip ex ea commodo consequat", "cupidatat non proident, sunt in culpa", "Lorem ipsum dolor sit amet, consectetur", "Excepteur sint occaecat cupidatat non proident", "sunt in culpa qui officia", "deserunt mollit anim id est laborum", "Duis aute irure dolor"}
|
||||
|
||||
var analysisregion = []string{"Layer 1", "Layer 2", "Layer 3", "Layer 4", "Layer 5", "Layer 6", "Layer 7", "Layer 8", "Layer 9", "Layer 10"}
|
||||
|
||||
var classifierType = []string{"A", "B", "C", "D"}
|
||||
|
||||
var classList = []string{"1", "2", "3", "4"}
|
||||
|
||||
var outputType = []string{"MASK", "MASK-RCNN", "MASK-NMS"}
|
||||
|
||||
var dye = []string{"DAPI", "Autofluorescence", "Opal 690", "Opal 650", "Opal 620", "Opal 570", "Opal 540", "Opal 520"}
|
||||
|
||||
var outputImage = []string{"Dye 1", "Dye 2", "Dye 3", "Dye 4", "Dye 5", "Dye 6", "Dye 7", "Dye 8"}
|
||||
|
||||
// Ensure MerckSource implements interface.
|
||||
var _ idk.Source = (*MerckSource)(nil)
|
||||
|
||||
// MerckSource is an instance of a source generated
|
||||
// by the Sourcer implementation Merck.
|
||||
type MerckSource struct {
|
||||
cur uint64
|
||||
endAt uint64
|
||||
Z *gen.Gen
|
||||
rand *rand.Rand
|
||||
algZipf *rand.Zipf
|
||||
maskZipf *rand.Zipf
|
||||
analysisregionZipf *rand.Zipf
|
||||
classListZipf *rand.Zipf
|
||||
classifierTypeZipf *rand.Zipf
|
||||
outputTypeZipf *rand.Zipf
|
||||
dyeZipf *rand.Zipf
|
||||
outputImageZipf *rand.Zipf
|
||||
speciesZipf *rand.Zipf
|
||||
sexZipf *rand.Zipf
|
||||
commentsZipf *rand.Zipf
|
||||
tissuetypeZipf *rand.Zipf
|
||||
locationTissueZipf *rand.Zipf
|
||||
morphologyZipf *rand.Zipf
|
||||
etiologyZipf *rand.Zipf
|
||||
procedures_descZipf *rand.Zipf
|
||||
diseaseZipf *rand.Zipf
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
// Record implements idk.Source.
|
||||
func (e *MerckSource) Record() (idk.Record, error) {
|
||||
if e.cur >= e.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
imageName := e.String(5) + ".tif"
|
||||
imageLocation := "\\myserver\\" + imageName
|
||||
|
||||
// ID allocation
|
||||
total_records := e.endAt - 1
|
||||
subjectID := int(math.Ceil(float64(total_records) / float64(128)))
|
||||
caseID := int(math.Ceil(float64(total_records) / float64(32)))
|
||||
specimenID := int(math.Ceil(float64(total_records) / float64(8)))
|
||||
|
||||
//<-- CELLS BEGIN -->
|
||||
dye_1_p := e.generate10(1000000, 0)
|
||||
dye_2_p := e.generate10(1000000, 0)
|
||||
dye_3_p := e.generate10(1000000, 0)
|
||||
dye_4_p := e.generate10(1000000, 0)
|
||||
dye_5_p := e.generate10(1000000, 0)
|
||||
dye_6_p := e.generate10(1000000, 0)
|
||||
dye_7_p := e.generate10(1000000, 0)
|
||||
dye_8_p := e.generate10(1000000, 0)
|
||||
dye_1_p_n := e.generate10(1000000, 0)
|
||||
dye_2_p_n := e.generate10(1000000, 0)
|
||||
dye_3_p_n := e.generate10(1000000, 0)
|
||||
dye_4_p_n := e.generate10(1000000, 0)
|
||||
dye_5_p_n := e.generate10(1000000, 0)
|
||||
dye_6_p_n := e.generate10(1000000, 0)
|
||||
dye_7_p_n := e.generate10(1000000, 0)
|
||||
dye_8_p_n := e.generate10(1000000, 0)
|
||||
dye_1_p_c := e.generate50(1000000, 0)
|
||||
dye_2_p_c := e.generate50(1000000, 0)
|
||||
dye_3_p_c := e.generate50(1000000, 0)
|
||||
dye_4_p_c := e.generate50(1000000, 0)
|
||||
dye_5_p_c := e.generate50(1000000, 0)
|
||||
dye_6_p_c := e.generate50(1000000, 0)
|
||||
dye_7_p_c := e.generate50(1000000, 0)
|
||||
dye_8_p_c := e.generate50(1000000, 0)
|
||||
dye_1_p_m := e.generate50(1000000, 0)
|
||||
dye_2_p_m := e.generate50(1000000, 0)
|
||||
dye_3_p_m := e.generate50(1000000, 0)
|
||||
dye_4_p_m := e.generate50(1000000, 0)
|
||||
dye_5_p_m := e.generate50(1000000, 0)
|
||||
dye_6_p_m := e.generate50(1000000, 0)
|
||||
dye_7_p_m := e.generate50(1000000, 0)
|
||||
dye_8_p_m := e.generate50(1000000, 0)
|
||||
total_cells := e.generateRandomInt(1000000, 1000)
|
||||
//<-- CELLS END -->
|
||||
|
||||
e.record[0] = e.cur
|
||||
e.record[1] = imageLocation
|
||||
e.record[2] = imageName
|
||||
e.record[3] = alg[e.algZipf.Uint64()]
|
||||
e.record[4] = startTime.Add(time.Duration(e.Z.R.Int63n(int64(timeSpan))))
|
||||
e.record[5] = analysisregion[e.analysisregionZipf.Uint64()]
|
||||
e.record[6] = total_cells
|
||||
e.record[7] = dye_1_p
|
||||
e.record[8] = dye_2_p
|
||||
e.record[9] = dye_3_p
|
||||
e.record[10] = dye_4_p
|
||||
e.record[11] = dye_5_p
|
||||
e.record[12] = dye_6_p
|
||||
e.record[13] = dye_7_p
|
||||
e.record[14] = dye_8_p
|
||||
e.record[15] = dye_1_p_n
|
||||
e.record[16] = dye_2_p_n
|
||||
e.record[17] = dye_3_p_n
|
||||
e.record[18] = dye_4_p_n
|
||||
e.record[19] = dye_5_p_n
|
||||
e.record[20] = dye_6_p_n
|
||||
e.record[21] = dye_7_p_n
|
||||
e.record[22] = dye_8_p_n
|
||||
e.record[23] = e.generate50(1000000, 0)
|
||||
e.record[24] = dye_1_p_c
|
||||
e.record[25] = dye_2_p_c
|
||||
e.record[26] = dye_3_p_c
|
||||
e.record[27] = dye_4_p_c
|
||||
e.record[28] = dye_5_p_c
|
||||
e.record[29] = dye_6_p_c
|
||||
e.record[30] = dye_7_p_c
|
||||
e.record[31] = dye_8_p_c
|
||||
e.record[32] = e.generate50(1000000, 0)
|
||||
e.record[33] = dye_1_p_m
|
||||
e.record[34] = dye_2_p_m
|
||||
e.record[35] = dye_3_p_m
|
||||
e.record[36] = dye_4_p_m
|
||||
e.record[37] = dye_5_p_m
|
||||
e.record[38] = dye_6_p_m
|
||||
e.record[39] = dye_7_p_m
|
||||
e.record[40] = dye_8_p_m
|
||||
e.record[41] = e.generate50(1000000, 0)
|
||||
e.record[42] = float64(dye_1_p) / float64(total_cells)
|
||||
e.record[43] = float64(dye_3_p) / float64(total_cells)
|
||||
e.record[44] = float64(dye_3_p) / float64(total_cells)
|
||||
e.record[45] = float64(dye_4_p) / float64(total_cells)
|
||||
e.record[46] = float64(dye_5_p) / float64(total_cells)
|
||||
e.record[47] = float64(dye_6_p) / float64(total_cells)
|
||||
e.record[48] = float64(dye_7_p) / float64(total_cells)
|
||||
e.record[49] = float64(dye_8_p) / float64(total_cells)
|
||||
e.record[50] = float64(dye_1_p_n) / float64(total_cells)
|
||||
e.record[51] = float64(dye_2_p_n) / float64(total_cells)
|
||||
e.record[52] = float64(dye_3_p_n) / float64(total_cells)
|
||||
e.record[53] = float64(dye_4_p_n) / float64(total_cells)
|
||||
e.record[54] = float64(dye_5_p_n) / float64(total_cells)
|
||||
e.record[55] = float64(dye_6_p_n) / float64(total_cells)
|
||||
e.record[56] = float64(dye_7_p_n) / float64(total_cells)
|
||||
e.record[57] = float64(dye_8_p_n) / float64(total_cells)
|
||||
e.record[58] = float64(dye_1_p_c) / float64(total_cells)
|
||||
e.record[59] = float64(dye_2_p_c) / float64(total_cells)
|
||||
e.record[60] = float64(dye_3_p_c) / float64(total_cells)
|
||||
e.record[61] = float64(dye_4_p_c) / float64(total_cells)
|
||||
e.record[62] = float64(dye_5_p_c) / float64(total_cells)
|
||||
e.record[63] = float64(dye_6_p_c) / float64(total_cells)
|
||||
e.record[64] = float64(dye_7_p_c) / float64(total_cells)
|
||||
e.record[65] = float64(dye_8_p_c) / float64(total_cells)
|
||||
e.record[66] = float64(dye_1_p_m) / float64(total_cells)
|
||||
e.record[67] = float64(dye_2_p_m) / float64(total_cells)
|
||||
e.record[68] = float64(dye_3_p_m) / float64(total_cells)
|
||||
e.record[69] = float64(dye_4_p_m) / float64(total_cells)
|
||||
e.record[70] = float64(dye_5_p_m) / float64(total_cells)
|
||||
e.record[71] = float64(dye_6_p_m) / float64(total_cells)
|
||||
e.record[72] = float64(dye_7_p_m) / float64(total_cells)
|
||||
e.record[73] = float64(dye_8_p_m) / float64(total_cells)
|
||||
e.record[74] = e.float50(50, 0)
|
||||
e.record[75] = e.float50(50, 0)
|
||||
e.record[76] = e.float50(50, 0)
|
||||
e.record[77] = e.float50(50, 0)
|
||||
e.record[78] = e.float50(50, 0)
|
||||
e.record[79] = e.float50(50, 0)
|
||||
e.record[80] = e.float50(50, 0)
|
||||
e.record[81] = e.float50(50, 0)
|
||||
e.record[82] = e.float50(50, 0)
|
||||
e.record[83] = e.float50(50, 0)
|
||||
e.record[84] = e.float50(50, 0)
|
||||
e.record[85] = e.float50(50, 0)
|
||||
e.record[86] = e.float50(50, 0)
|
||||
e.record[87] = e.float50(50, 0)
|
||||
e.record[88] = e.float50(50, 0)
|
||||
e.record[89] = e.float50(50, 0)
|
||||
e.record[90] = e.float50(50, 0)
|
||||
e.record[91] = e.float50(50, 0)
|
||||
e.record[92] = e.float50(50, 0)
|
||||
e.record[93] = e.float50(50, 0)
|
||||
e.record[94] = e.float50(50, 0)
|
||||
e.record[95] = e.float50(50, 0)
|
||||
e.record[96] = e.float50(50, 0)
|
||||
e.record[97] = e.float50(50, 0)
|
||||
e.record[98] = e.floatRand(500, 5)
|
||||
e.record[99] = e.floatRand(500, 5)
|
||||
e.record[100] = e.floatRand(500, 5)
|
||||
e.record[101] = e.generateRandomInt(1000000000, 10000000)
|
||||
e.record[102] = e.generateRandomInt(5, 1)
|
||||
e.record[103] = e.Z.R.Intn(2) == 0
|
||||
e.record[104] = classList[e.classListZipf.Uint64()]
|
||||
e.record[105] = e.Z.R.Intn(2) == 0
|
||||
e.record[106] = outputType[e.outputTypeZipf.Uint64()]
|
||||
e.record[107] = dye[e.dyeZipf.Uint64()]
|
||||
e.record[108] = e.Z.R.Intn(2) == 0
|
||||
e.record[109] = e.floatRand(1, 0)
|
||||
e.record[110] = e.floatRand(1, 0)
|
||||
e.record[111] = e.floatRand(1, 0)
|
||||
e.record[112] = e.floatRand(1, 0)
|
||||
e.record[113] = mask[e.maskZipf.Uint64()]
|
||||
e.record[114] = mask[e.maskZipf.Uint64()]
|
||||
e.record[115] = mask[e.maskZipf.Uint64()]
|
||||
e.record[116] = dye[e.dyeZipf.Uint64()]
|
||||
e.record[117] = e.Z.R.Intn(2) == 0
|
||||
e.record[118] = e.floatRand(1, 0)
|
||||
e.record[119] = e.floatRand(1, 0)
|
||||
e.record[120] = e.floatRand(1, 0)
|
||||
e.record[121] = e.floatRand(1, 0)
|
||||
e.record[122] = mask[e.maskZipf.Uint64()]
|
||||
e.record[123] = mask[e.maskZipf.Uint64()]
|
||||
e.record[124] = mask[e.maskZipf.Uint64()]
|
||||
e.record[125] = dye[e.dyeZipf.Uint64()]
|
||||
e.record[126] = e.Z.R.Intn(2) == 0
|
||||
e.record[127] = e.floatRand(1, 0)
|
||||
e.record[128] = e.floatRand(1, 0)
|
||||
e.record[129] = e.floatRand(1, 0)
|
||||
e.record[130] = e.floatRand(1, 0)
|
||||
e.record[131] = mask[e.maskZipf.Uint64()]
|
||||
e.record[132] = mask[e.maskZipf.Uint64()]
|
||||
e.record[133] = mask[e.maskZipf.Uint64()]
|
||||
e.record[134] = dye[e.dyeZipf.Uint64()]
|
||||
e.record[135] = e.Z.R.Intn(2) == 0
|
||||
e.record[136] = e.floatRand(1, 0)
|
||||
e.record[137] = e.floatRand(1, 0)
|
||||
e.record[138] = e.floatRand(1, 0)
|
||||
e.record[139] = e.floatRand(1, 0)
|
||||
e.record[140] = mask[e.maskZipf.Uint64()]
|
||||
e.record[141] = mask[e.maskZipf.Uint64()]
|
||||
e.record[142] = mask[e.maskZipf.Uint64()]
|
||||
e.record[143] = dye[e.dyeZipf.Uint64()]
|
||||
e.record[144] = e.Z.R.Intn(2) == 0
|
||||
e.record[145] = e.floatRand(1, 0)
|
||||
e.record[146] = e.floatRand(1, 0)
|
||||
e.record[147] = e.floatRand(1, 0)
|
||||
e.record[148] = e.floatRand(1, 0)
|
||||
e.record[149] = mask[e.maskZipf.Uint64()]
|
||||
e.record[150] = mask[e.maskZipf.Uint64()]
|
||||
e.record[151] = mask[e.maskZipf.Uint64()]
|
||||
e.record[152] = dye[e.dyeZipf.Uint64()]
|
||||
e.record[153] = e.Z.R.Intn(2) == 0
|
||||
e.record[154] = e.floatRand(1, 0)
|
||||
e.record[155] = e.floatRand(1, 0)
|
||||
e.record[156] = e.floatRand(1, 0)
|
||||
e.record[157] = e.floatRand(1, 0)
|
||||
e.record[158] = mask[e.maskZipf.Uint64()]
|
||||
e.record[159] = mask[e.maskZipf.Uint64()]
|
||||
e.record[160] = mask[e.maskZipf.Uint64()]
|
||||
e.record[161] = dye[e.dyeZipf.Uint64()]
|
||||
e.record[162] = e.Z.R.Intn(2) == 0
|
||||
e.record[163] = e.floatRand(1, 0)
|
||||
e.record[164] = e.floatRand(1, 0)
|
||||
e.record[165] = e.floatRand(1, 0)
|
||||
e.record[166] = e.floatRand(1, 0)
|
||||
e.record[167] = mask[e.maskZipf.Uint64()]
|
||||
e.record[168] = mask[e.maskZipf.Uint64()]
|
||||
e.record[169] = mask[e.maskZipf.Uint64()]
|
||||
e.record[170] = dye[e.dyeZipf.Uint64()]
|
||||
e.record[171] = e.Z.R.Intn(2) == 0
|
||||
e.record[172] = e.floatRand(1, 0)
|
||||
e.record[173] = e.floatRand(1, 0)
|
||||
e.record[174] = e.floatRand(1, 0)
|
||||
e.record[175] = e.floatRand(1, 0)
|
||||
e.record[176] = mask[e.maskZipf.Uint64()]
|
||||
e.record[177] = mask[e.maskZipf.Uint64()]
|
||||
e.record[178] = mask[e.maskZipf.Uint64()]
|
||||
e.record[179] = e.floatRand(1, 0)
|
||||
e.record[180] = e.floatRand(1, 0)
|
||||
e.record[181] = e.floatRand(1, 0)
|
||||
e.record[182] = e.floatRand(1, 0)
|
||||
e.record[183] = e.floatRand(50, 0)
|
||||
e.record[184] = e.floatRand(1000, 100)
|
||||
e.record[185] = e.floatRand(0.5, 0)
|
||||
e.record[186] = e.floatRand(100, 1)
|
||||
e.record[187] = e.floatRand(0.5, 0)
|
||||
e.record[188] = e.floatRand(10, 0)
|
||||
e.record[189] = e.floatRand(1000, 15)
|
||||
e.record[190] = outputImage[e.outputImageZipf.Uint64()]
|
||||
e.record[191] = e.generateRandomInt(1, 0)
|
||||
e.record[192] = e.floatRand(50, 0)
|
||||
e.record[193] = e.floatRand(1000, 100)
|
||||
e.record[194] = e.floatRand(50, 0)
|
||||
e.record[195] = e.floatRand(1000, 100)
|
||||
e.record[196] = e.generateRandomInt(subjectID, 0)
|
||||
e.record[197] = species[e.speciesZipf.Uint64()]
|
||||
e.record[198] = sex[e.sexZipf.Uint64()]
|
||||
e.record[199] = startTime.Add(time.Duration(e.Z.R.Int63n(int64(timeSpan))))
|
||||
e.record[200] = e.generateRandomInt(caseID, 0)
|
||||
e.record[201] = e.generateRandomInt(95, 0)
|
||||
e.record[202] = e.generateRandomInt(200, 1)
|
||||
set := e.generateRandomInt(10, 0)
|
||||
procedureIDs := make([]uint64, 0, set)
|
||||
procedureNames := make([]string, 0, set)
|
||||
procedureDesc := make([]string, 0, set)
|
||||
for v := 0; v < set; v++ {
|
||||
procedureIDs = append(procedureIDs, uint64(e.generateRandomInt(10000, 0)))
|
||||
procedureNames = append(procedureNames, "P"+strconv.Itoa(v))
|
||||
procedureDesc = append(procedureDesc, procedures_desc[e.procedures_descZipf.Uint64()])
|
||||
}
|
||||
e.record[203] = procedureIDs
|
||||
e.record[204] = procedureNames
|
||||
e.record[205] = procedureDesc
|
||||
e.record[206] = e.generateRandomInt(10000, 1)
|
||||
e.record[207] = e.generateRandomInt(20000, 1)
|
||||
e.record[208] = comments[e.commentsZipf.Uint64()]
|
||||
e.record[209] = e.generateRandomInt(10000, 1)
|
||||
e.record[210] = disease[e.diseaseZipf.Uint64()]
|
||||
e.record[211] = etiology[e.etiologyZipf.Uint64()]
|
||||
e.record[212] = morphology[e.morphologyZipf.Uint64()]
|
||||
e.record[213] = e.generateRandomInt(10000, 1)
|
||||
e.record[214] = e.generateRandomInt(specimenID, 0)
|
||||
e.record[215] = tissuetype[e.tissuetypeZipf.Uint64()]
|
||||
e.record[216] = e.generateRandomInt(500, 1)
|
||||
e.record[217] = locationTissue[e.locationTissueZipf.Uint64()]
|
||||
e.cur++
|
||||
|
||||
return e.record, nil
|
||||
}
|
||||
|
||||
// Schema implements idk.Source.
|
||||
func (e *MerckSource) Schema() []idk.Field {
|
||||
return e.schema
|
||||
}
|
||||
|
||||
func (e *MerckSource) Seed(seed int64) {
|
||||
e.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*MerckSource)(nil)
|
||||
|
||||
const charset = "abcdefghijklmnopqrstuvwxyz" +
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
|
||||
// returns a string with random characters from a predefined charset of a specified length
|
||||
func (e *MerckSource) StringWithCharset(length int, charset string) string {
|
||||
b := make([]byte, length)
|
||||
for i := range b {
|
||||
b[i] = charset[e.rand.Intn(len(charset))]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
//returns a string with random English alphabets of the specified length
|
||||
func (e *MerckSource) String(length int) string {
|
||||
return e.StringWithCharset(length, charset)
|
||||
}
|
||||
|
||||
//returns a random int between a range with a 50% chance of returning a zero.
|
||||
func (e *MerckSource) generate50(max int, min int) int {
|
||||
num := e.rand.Intn(100)
|
||||
numbertoReturn := e.rand.Intn(max-min) + min
|
||||
switch {
|
||||
case num < 50:
|
||||
return 0
|
||||
default:
|
||||
return numbertoReturn
|
||||
}
|
||||
}
|
||||
|
||||
//returns a random int between a range with a 10% chance of returning a zero.
|
||||
func (e *MerckSource) generate10(max int, min int) int {
|
||||
num := e.rand.Intn(100)
|
||||
numbertoReturn := e.rand.Intn(max-min) + min
|
||||
switch {
|
||||
case num < 10:
|
||||
return 0
|
||||
default:
|
||||
return numbertoReturn
|
||||
}
|
||||
}
|
||||
|
||||
//returns a random float between a range with a 50% chance of returning a zero.
|
||||
func (e *MerckSource) float50(max float64, min float64) float64 {
|
||||
numbertoReturn := min + e.rand.Float64()*(max-min)
|
||||
num := e.rand.Intn(100)
|
||||
switch {
|
||||
case num < 50:
|
||||
return 0
|
||||
default:
|
||||
return numbertoReturn
|
||||
}
|
||||
}
|
||||
|
||||
//returns a random float between a range
|
||||
func (e *MerckSource) floatRand(max float64, min float64) float64 {
|
||||
return min + e.rand.Float64()*(max-min)
|
||||
}
|
||||
|
||||
//returns a random int between a range
|
||||
func (e *MerckSource) generateRandomInt(max int, min int) int {
|
||||
return e.rand.Intn(max-min) + min
|
||||
}
|
||||
|
||||
func (e *MerckSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
248
idk/datagen/network.go
Normal file
248
idk/datagen/network.go
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Network implements interface.
|
||||
var _ Sourcer = (*Network)(nil)
|
||||
|
||||
// Network implements Sourcer.
|
||||
type Network struct {
|
||||
schema []idk.Field
|
||||
}
|
||||
|
||||
// NewNetwork returns a new instance of Network.
|
||||
func NewNetwork(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Network{
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.IDField{NameVal: "qos_tier"}, // 1
|
||||
idk.IntField{NameVal: "source_equip_id"}, // 2
|
||||
idk.IntField{NameVal: "dest_equip_id"}, // 3
|
||||
idk.IntField{NameVal: "data_size"}, // 4
|
||||
idk.StringField{NameVal: "data_type"}, // 5
|
||||
idk.StringField{NameVal: "customer"}, // 6
|
||||
idk.IntField{NameVal: "timestamp"}, // 7
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (n *Network) Source(cfg SourceConfig) idk.Source {
|
||||
src := &NetworkSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
|
||||
schema: n.schema,
|
||||
}
|
||||
|
||||
src.typeZipf = rand.NewZipf(src.rand, 1.03, 4, uint64(len(dataTypeStrings))-1)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[7] = int(1420070400 + cfg.startFrom)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (n *Network) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (n *Network) DefaultEndAt() uint64 {
|
||||
return 100000000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (n *Network) Info() string {
|
||||
return "Generates timeseries network transmission data (from_ip, to_ip, bytes) - references data in sites, manufacturer, and equipment."
|
||||
}
|
||||
|
||||
// Ensure NetworkSource implements interface.
|
||||
var _ idk.Source = (*NetworkSource)(nil)
|
||||
|
||||
type NetworkSource struct {
|
||||
Log logger.Logger
|
||||
|
||||
rand *rand.Rand
|
||||
typeZipf *rand.Zipf
|
||||
|
||||
schema []idk.Field
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
record record
|
||||
}
|
||||
|
||||
func (s *NetworkSource) Record() (idk.Record, error) {
|
||||
// ID
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.cur
|
||||
s.cur++
|
||||
|
||||
// QOS
|
||||
s.record[1] = s.qos()
|
||||
|
||||
// TODO - equipment et. al
|
||||
dataType := s.dataType()
|
||||
s.record[5] = dataType
|
||||
|
||||
s.record[2] = s.srcEquip(dataType)
|
||||
s.record[3] = s.dstEquip(dataType)
|
||||
|
||||
s.record[4] = s.size(dataTypes[dataType])
|
||||
|
||||
s.record[6] = s.customer(dataType)
|
||||
|
||||
// timestamp
|
||||
s.record[7] = s.record[7].(int) + s.rand.Intn(3)
|
||||
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *NetworkSource) customer(dataType string) string {
|
||||
if dataType == "Transaction" {
|
||||
return customers[s.rand.Intn(4)]
|
||||
}
|
||||
return customers[s.rand.Intn(len(customers))]
|
||||
}
|
||||
|
||||
func (s *NetworkSource) size(dt dataType) int64 {
|
||||
return s.rand.Int63n(dt.maxSize-dt.minSize) + dt.minSize
|
||||
}
|
||||
|
||||
type dataType struct {
|
||||
minSize int64
|
||||
maxSize int64
|
||||
}
|
||||
|
||||
var dataTypes = map[string]dataType{
|
||||
"Wireless Internet": {
|
||||
minSize: 51,
|
||||
maxSize: 22000000, // 22MB
|
||||
},
|
||||
"Wired Internet": {
|
||||
minSize: 63,
|
||||
maxSize: 40000000, // 40MB
|
||||
},
|
||||
"Wired Video": {
|
||||
minSize: 73,
|
||||
maxSize: 150000000, // 150MB
|
||||
},
|
||||
"Transaction": {
|
||||
minSize: 12,
|
||||
maxSize: 1200, // 7MB
|
||||
},
|
||||
"Wireless Video": {
|
||||
minSize: 75,
|
||||
maxSize: 50000000, // 50MB
|
||||
},
|
||||
"Wireless Voice": {
|
||||
minSize: 17,
|
||||
maxSize: 7000000, // 7MB
|
||||
},
|
||||
"Wired Voice": {
|
||||
minSize: 19,
|
||||
maxSize: 10000000, // 10MB
|
||||
},
|
||||
}
|
||||
var dataTypeStrings []string
|
||||
|
||||
func init() {
|
||||
for name := range dataTypes {
|
||||
dataTypeStrings = append(dataTypeStrings, name)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *NetworkSource) dataType() string {
|
||||
return dataTypeStrings[s.typeZipf.Uint64()]
|
||||
}
|
||||
|
||||
func (s *NetworkSource) srcEquip(typ string) int64 {
|
||||
switch typ {
|
||||
case "Transaction":
|
||||
return s.rand.Int63n(1000000) + 35000000 // POS device
|
||||
case "Wired Video":
|
||||
return s.rand.Int63n(28000000) + 5000000 // all non BTS devices
|
||||
case "Wired Internet":
|
||||
return s.rand.Int63n(28000000) + 5000000 // all non BTS devices
|
||||
case "Wired Voice":
|
||||
return s.rand.Int63n(28000000) + 5000000 // all non BTS devices
|
||||
case "Wireless Video":
|
||||
return s.rand.Int63n(17000000) + 18000000 // all non CPE devices
|
||||
case "Wireless Internet":
|
||||
return s.rand.Int63n(17000000) + 18000000 // all non CPE devices
|
||||
case "Wireless Voice":
|
||||
return s.rand.Int63n(17000000) + 18000000 // all non CPE devices
|
||||
default:
|
||||
panic("srcEquip: unknown type " + typ)
|
||||
}
|
||||
}
|
||||
func (s *NetworkSource) dstEquip(typ string) int64 {
|
||||
switch typ {
|
||||
case "Transaction":
|
||||
return s.rand.Int63n(15000000) + 18000000 // server and router
|
||||
case "Wired Video":
|
||||
return s.rand.Int63n(28000000) + 5000000 // all non BTS devices
|
||||
case "Wired Internet":
|
||||
return s.rand.Int63n(28000000) + 5000000 // all non BTS devices
|
||||
case "Wired Voice":
|
||||
return s.rand.Int63n(28000000) + 5000000 // all non BTS devices
|
||||
case "Wireless Video":
|
||||
return s.rand.Int63n(17000000) + 18000000 // all non CPE devices
|
||||
case "Wireless Internet":
|
||||
return s.rand.Int63n(17000000) + 18000000 // all non CPE devices
|
||||
case "Wireless Voice":
|
||||
return s.rand.Int63n(17000000) + 18000000 // all non CPE devices
|
||||
default:
|
||||
panic("dstEquip: unknown type " + typ)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *NetworkSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *NetworkSource) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*NetworkSource)(nil)
|
||||
|
||||
// qos returns a QoS value from 1-5 with a 5%/25%/40%/25%/5% weighting.
|
||||
func (s *NetworkSource) qos() uint64 {
|
||||
num := s.rand.Intn(100)
|
||||
switch {
|
||||
case num < 5:
|
||||
return 1
|
||||
case num < 30:
|
||||
return 2
|
||||
case num < 70:
|
||||
return 3
|
||||
case num < 95:
|
||||
return 4
|
||||
default:
|
||||
return 5
|
||||
}
|
||||
}
|
||||
|
||||
func (s *NetworkSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
348
idk/datagen/palo_alto.go
Normal file
348
idk/datagen/palo_alto.go
Normal file
|
|
@ -0,0 +1,348 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
"github.com/molecula/featurebase/v3/pql"
|
||||
)
|
||||
|
||||
// Ensure PaloAlto implements interface.
|
||||
var _ Sourcer = (*PaloAlto)(nil)
|
||||
|
||||
// PaloAlto implements Sourcer.
|
||||
type PaloAlto struct {
|
||||
schema []idk.Field
|
||||
}
|
||||
|
||||
// NewPaloAlto returns a new instance of PaloAlto.
|
||||
func NewPaloAlto(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &PaloAlto{
|
||||
schema: []idk.Field{
|
||||
idk.StringField{NameVal: "vlan_source"}, // 0) "unknown"
|
||||
idk.StringField{NameVal: "severity_level"}, // 1) "Normal"
|
||||
idk.StringField{NameVal: "display_profile_type"}, // 2) "IoT"
|
||||
idk.BoolField{NameVal: "keep"}, // 3) true
|
||||
idk.StringField{NameVal: "internal_risk_level"}, // 4) "Low"
|
||||
idk.IntField{NameVal: "ml_progress"}, // 5) 100
|
||||
idk.StringField{NameVal: "tenantid"}, // 6) "gjtu7ldl"
|
||||
idk.StringField{NameVal: "display_oscombined"}, // 7) "Windows CE"
|
||||
idk.StringField{NameVal: "profileid"}, // 8) "Zebra Technologies Tracking Device"
|
||||
idk.StringField{NameVal: "risk_main_contributors"}, // 9) "Application Usage"
|
||||
idk.IntField{NameVal: "profile_date"}, // 10) "2019-03-14T06:09:00.000Z"
|
||||
idk.StringField{NameVal: "risk_level"}, // 11) "low"
|
||||
idk.StringArrayField{NameVal: "baseline-learning_category"}, // 12) "payload"
|
||||
idk.StringField{NameVal: "baseline-state"}, // 13) "stable"
|
||||
idk.StringField{NameVal: "baseline-protected"}, // 14) "profile model"
|
||||
idk.IntField{NameVal: "baseline-progress"}, // 15) 100
|
||||
idk.IntField{NameVal: "baseline-pi"}, // 16) 98
|
||||
idk.StringArrayField{NameVal: "baseline-invalid_category"}, // 17) "ext_inbound", "ext_outbound"
|
||||
idk.DecimalField{NameVal: "baseline-anomaly_score-application", Scale: 1}, // 18) 0.1
|
||||
idk.DecimalField{NameVal: "baseline-anomaly_score-internal", Scale: 4}, // 19) 0.4097
|
||||
idk.DecimalField{NameVal: "baseline-anomaly_score-protocol", Scale: 4}, // 20) 0.3736
|
||||
idk.DecimalField{NameVal: "baseline-anomaly_score-external", Scale: 4}, // 21) 0.3548
|
||||
idk.DecimalField{NameVal: "baseline-anomaly_score-payload", Scale: 4}, // 22) 0.2944
|
||||
idk.IntField{NameVal: "profile_confidence"}, // 23) 86
|
||||
idk.StringField{NameVal: "display_osgroup"}, // 24) "Windows CE"
|
||||
idk.IntField{NameVal: "ml_risk-risk_score"}, // 25) 10
|
||||
idk.IntField{NameVal: "ml_risk-progress"}, // 26) 100
|
||||
idk.StringField{NameVal: "ml_risk-severity_level"}, // 27) "Low"
|
||||
idk.DecimalField{NameVal: "ml_risk-anomaly_score-application", Scale: 2}, // 28) 0.25
|
||||
idk.DecimalField{NameVal: "ml_risk-anomaly_score-internal", Scale: 2}, // 29) 0.25
|
||||
idk.DecimalField{NameVal: "ml_risk-anomaly_score-protocol", Scale: 2}, // 30) 0.25
|
||||
idk.DecimalField{NameVal: "ml_risk-anomaly_score-external", Scale: 2}, // 31) 0.25
|
||||
idk.DecimalField{NameVal: "ml_risk-anomaly_score-payload", Scale: 2}, // 32) 0.25
|
||||
idk.IntField{NameVal: "ml_risk-date"}, // 33) "2020-09-22T00:00:00Z"
|
||||
idk.StringField{NameVal: "risk_level_source"}, // 34) "offline_analytics"
|
||||
idk.IntField{NameVal: "risk_score"}, // 35) 40
|
||||
idk.StringField{NameVal: "source"}, // 36) ""
|
||||
idk.IntField{NameVal: "internal_risk_score"}, // 37) 10
|
||||
idk.StringField{NameVal: "connect_evtcontent-username"}, // 38) "unknown"
|
||||
idk.StringField{NameVal: "connect_evtcontent-hostname"}, // 39) "unknown"
|
||||
idk.BoolField{NameVal: "connect_evtcontent-monitored"}, // 40) true
|
||||
idk.StringField{NameVal: "connect_evtcontent-ossource"}, // 41) "inspector"
|
||||
idk.StringField{NameVal: "connect_evtcontent-roles"}, // 42) ""
|
||||
idk.StringField{NameVal: "connect_evtcontent-osgroup"}, // 43) "Embedded"
|
||||
idk.StringField{NameVal: "connect_evtcontent-ip"}, // 44) "10.2.171.9"
|
||||
idk.IntField{NameVal: "connect_evtcontent-vlan"}, // 45) 0
|
||||
idk.StringField{NameVal: "connect_evtcontent-useragent"}, // 46) "unknown"
|
||||
idk.StringField{NameVal: "connect_evtcontent-os"}, // 47) "Palm OS"
|
||||
idk.StringField{NameVal: "connect_evtcontent-osver_src"}, // 48) "offline"
|
||||
idk.StringField{NameVal: "display_vendor"}, // 49) "Zebra Technologies"
|
||||
idk.StringField{NameVal: "profile_vendor"}, // 50) "Zebra Technologies"
|
||||
idk.IntField{NameVal: "ml_risk_score"}, // 51) 10
|
||||
idk.StringField{NameVal: "subnets"}, // 52) "10.2.170.0/23"
|
||||
idk.StringField{NameVal: "ml_risk_level"}, // 53) "Low"
|
||||
idk.StringField{NameVal: "profile_type"}, // 54) "IoT"
|
||||
idk.IntField{NameVal: "profile_type_score"}, // 55) 100
|
||||
idk.StringField{NameVal: "display_hostname"}, // 56) "unknown"
|
||||
idk.StringField{NameVal: "display_os"}, // 57) "Palm OS"
|
||||
idk.StringField{NameVal: "vendor"}, // 58) "Zebra Technologies"
|
||||
idk.StringField{NameVal: "siteid"}, // 59) "6"
|
||||
idk.IntField{NameVal: "progress"}, // 60) 100
|
||||
idk.IntField{NameVal: "date"}, // 61) "2019-02-09T03:31:08.590Z"
|
||||
idk.IntField{NameVal: "risk_date"}, // 62) "2018-10-28T00:41:17.534012Z"
|
||||
idk.StringField{NameVal: "display_vlan"}, // 63) "unknown"
|
||||
idk.IntField{NameVal: "filter_last_modified"}, // 64) "2017-08-05T01:35:35.429Z"
|
||||
idk.StringField{NameVal: "profile_category"}, // 65) "Patient Tracking"
|
||||
idk.StringField{NameVal: "applianceid"}, // 66) "000000000000000000000CC47ADB6556"
|
||||
idk.StringField{NameVal: "profile_vertical"}, // 67) "Medical"
|
||||
idk.DecimalField{NameVal: "anomaly_score-application", Scale: 4}, // 68) 0.1442
|
||||
idk.DecimalField{NameVal: "anomaly_score-internal", Scale: 4}, // 69) 0.1587
|
||||
idk.DecimalField{NameVal: "anomaly_score-protocol", Scale: 4}, // 70) 0.11750000000000001
|
||||
idk.DecimalField{NameVal: "anomaly_score-payload", Scale: 1}, // 71) 0.1
|
||||
idk.DecimalField{NameVal: "anomaly_score-external", Scale: 1}, // 72) 0.1
|
||||
idk.StringField{NameVal: "profile_osgroup"}, // 73) "Windows CE"
|
||||
idk.StringField{NameVal: "user_profile_type"}, // 74) "IoT"
|
||||
idk.StringField{NameVal: "display_dhcp"}, // 75) "Yes"
|
||||
idk.StringField{NameVal: "profile_type_source"}, // 76) "profiler"
|
||||
idk.IntField{NameVal: "baseline_update_date"}, // 77) "2019.04.20"
|
||||
idk.StringField{NameVal: "deviceid"}, // 78) "40:83:de:9e:7d:8c"
|
||||
idk.StringField{NameVal: "profile_classifier"}, // 79) "OuiClassifier"
|
||||
idk.StringField{NameVal: "dhcp"}, // 80) "Yes"
|
||||
idk.StringField{NameVal: "display_profile_category"}, // 81) "Patient Tracking"
|
||||
idk.StringField{NameVal: "display_profileid"}, // 82) "Macbook Pro"
|
||||
idk.BoolField{NameVal: "unique"}, // 83) true
|
||||
idk.BoolField{NameVal: "pii"}, // 84) true
|
||||
idk.IntField{NameVal: "pii_ts"}, // 85) "2019-01-01T20:15:07.294Z"
|
||||
idk.StringField{NameVal: "display_osverfirmwarever"}, // 86) ""
|
||||
idk.StringField{NameVal: "display_wirewireless"}, // 87) "wireless"
|
||||
idk.StringField{NameVal: "wirewireless"}, // 88) "wireless"
|
||||
idk.StringField{NameVal: "display_epp_safety"}, // 89) "not_protected"
|
||||
idk.StringField{NameVal: "mac"}, // 90) "40:83:de:9e:7d:8c"
|
||||
idk.StringField{NameVal: "foreignaccess"}, // 91) "No"
|
||||
idk.DecimalField{NameVal: "ai_vendor_confidence", Scale: 2}, // 92) 98.52
|
||||
idk.IntField{NameVal: "subnets_date"}, // 93) "2020-09-17T18:42:28.691Z"
|
||||
idk.StringField{NameVal: "display_profile_confidence_source"}, // 94) "profiler"
|
||||
idk.IntField{NameVal: "display_profile_confidence_date"}, // 95) "2019-03-14T06:09:00Z"
|
||||
idk.IntField{NameVal: "display_profile_confidence"}, // 96) 86
|
||||
idk.StringField{NameVal: "display_profile_confidence_level"}, // 97) "70_Medium"
|
||||
idk.BoolField{NameVal: "filtered"}, // 98)
|
||||
idk.BoolField{NameVal: "routed"}, // 99)
|
||||
idk.StringField{NameVal: "object_id"}, // 100)
|
||||
idk.StringField{NameVal: "country_access_country_code"}, // 101)
|
||||
idk.BoolField{NameVal: "country_access_malicious"}, // 102)
|
||||
idk.IntField{NameVal: "country_access_ts"}, // 103)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (k *PaloAlto) Source(cfg SourceConfig) idk.Source {
|
||||
|
||||
src := &PaloAltoSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
schema: k.schema,
|
||||
}
|
||||
|
||||
src.g = gen.New(gen.OptGenSeed(cfg.seed))
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[17] = []string{}
|
||||
src.record[5] = make([]byte, 12)
|
||||
src.record[78] = []byte("40:83:de:9e:7d:8c")
|
||||
src.record[90] = []byte("40:83:de:9e:7d:8c")
|
||||
|
||||
// add object IDs
|
||||
// there'll be 10,000 unique Object IDs
|
||||
numUniqueObjectIDs := 10000
|
||||
objectIDLen := 10
|
||||
charset := "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
objectIDs := make([]string, numUniqueObjectIDs)
|
||||
for i := 0; i < numUniqueObjectIDs; i++ {
|
||||
randID := make([]byte, objectIDLen)
|
||||
for j := range randID {
|
||||
randID[j] = charset[src.g.R.Intn(len(charset))]
|
||||
}
|
||||
objectIDs[i] = string(randID)
|
||||
}
|
||||
src.objectIDs = objectIDs
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (k *PaloAlto) PrimaryKeyFields() []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (k *PaloAlto) DefaultEndAt() uint64 {
|
||||
return 20000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (k *PaloAlto) Info() string {
|
||||
return "Generates data representative of Palo Alto Networks' IoT/medical device security operations. 105 fields of various types, unkeyed."
|
||||
}
|
||||
|
||||
// Ensure PaloAltoSource implements interface.
|
||||
var _ idk.Source = (*PaloAltoSource)(nil)
|
||||
|
||||
// PaloAltoSource is a data generator which generates
|
||||
// data for all Pilosa field types.
|
||||
type PaloAltoSource struct {
|
||||
g *gen.Gen
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
|
||||
objectIDs []string
|
||||
}
|
||||
|
||||
func (k *PaloAltoSource) Record() (idk.Record, error) {
|
||||
if k.cur >= k.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
company := k.g.StringFromListWeighted(gen.Animals) + " " + k.g.StringFromListWeighted([]string{"Technologies", "Systems", "Associates"})
|
||||
os := k.g.StringFromListWeighted([]string{"Palm OS", "OSI", "FreeBSD", "BeOS"})
|
||||
baseTime := time.Date(2014, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
curTime := time.Date(2020, time.November, 1, 0, 0, 0, 0, time.UTC)
|
||||
durationSeconds := curTime.Sub(baseTime) / time.Second
|
||||
|
||||
// Increment the ID.
|
||||
k.cur++
|
||||
k.record[0] = k.g.StringFromListWeighted([]string{"unknown", "switch", "router", "gateway"})
|
||||
k.record[1] = k.g.StringFromListWeighted([]string{"Normal", "Low", "Zero", "Elevated", "High", "Critical"})
|
||||
k.record[2] = k.g.StringFromListWeighted([]string{"IoT", "Fixed", "Server"})
|
||||
k.record[3] = k.g.R.Intn(2) == 0
|
||||
k.record[4] = k.g.StringFromListWeighted([]string{"Low", "Normal", "Zero", "Elevated", "High", "Critical"})
|
||||
k.record[5] = k.g.R.Int63n(101)
|
||||
k.record[6] = k.g.StringFromListWeighted([]string{"gjtu7ldl", "C6RKKn", "9pFooX", "SvNzWU", "jCdISA", "y4fsPI", "ID5qFL", "zMpvSO", "DR4pCh",
|
||||
"5BbmPh", "gjTU0F", "DclwrO", "dpsZkt", "5vGgIy", "FQzIDE", "TW8d5P",
|
||||
"sxKStY", "1YULKk", "Gpn6r9", "HXy0Ju", "2EVpvj", "3wOt4J", "iXkNkv"})
|
||||
k.record[7] = k.g.StringFromListWeighted([]string{"Windows CE", "Windows 2000", "Windows XP", "Windows 95", "Mac OS 8", "Solaris", "MS Bob", "Plan 9", "Windows ME Professional Plus Ultimate Turbo Edition"})
|
||||
k.record[8] = company + " " + k.g.StringFromListWeighted([]string{"Tracking Device", "Pointing Device", "Input Device", "Display", ""})
|
||||
k.record[9] = k.g.StringFromListWeighted([]string{"Application Usage", "Automated Processes", "Background Scans"})
|
||||
k.record[10] = baseTime.Unix() + k.g.R.Int63n(int64(durationSeconds))
|
||||
k.record[11] = k.g.StringFromListWeighted([]string{"low", "normal", "zero", "elevated", "high", "critical"})
|
||||
k.record[12] = "payload"
|
||||
k.record[13] = k.g.StringFromListWeighted([]string{"stable", "unstable"})
|
||||
k.record[14] = "profile model"
|
||||
k.record[15] = k.g.R.Int63n(101)
|
||||
k.record[16] = k.g.R.Int63n(101)
|
||||
k.record[17] = k.g.StringSliceFromList(k.record[17].([]string), []string{"ext_inbound", "ext_outbound", "int_over", "int_under", "ext_across", "int_through"})
|
||||
k.record[18] = pql.NewDecimal(k.g.R.Int63n(10), 1)
|
||||
k.record[19] = pql.NewDecimal(k.g.R.Int63n(10000), 4)
|
||||
k.record[20] = pql.NewDecimal(k.g.R.Int63n(10000), 4)
|
||||
k.record[21] = pql.NewDecimal(k.g.R.Int63n(10000), 4)
|
||||
k.record[22] = pql.NewDecimal(k.g.R.Int63n(10000), 4)
|
||||
k.record[23] = k.g.R.Int63n(100)
|
||||
k.record[24] = k.g.StringFromListWeighted([]string{"Windows CE", "Windows 2000", "Windows XP", "Windows 95", "Mac OS 8", "Solaris", "MS Bob", "Plan 9", "Windows ME Professional Plus Ultimate Turbo Edition"})
|
||||
k.record[25] = k.g.R.Int63n(100)
|
||||
k.record[26] = k.g.R.Int63n(101)
|
||||
k.record[27] = k.g.StringFromListWeighted([]string{"Low", "Normal", "Zero", "Elevated", "High", "Critical"})
|
||||
k.record[28] = pql.NewDecimal(k.g.R.Int63n(100), 2) // 0.25
|
||||
k.record[29] = pql.NewDecimal(k.g.R.Int63n(100), 2) // 0.25
|
||||
k.record[30] = pql.NewDecimal(k.g.R.Int63n(100), 2) // 0.25
|
||||
k.record[31] = pql.NewDecimal(k.g.R.Int63n(100), 2) // 0.25
|
||||
k.record[32] = pql.NewDecimal(k.g.R.Int63n(100), 2) // 0.25
|
||||
k.record[33] = baseTime.Unix() + k.g.R.Int63n(int64(durationSeconds))
|
||||
k.record[34] = "offline_analytics"
|
||||
k.record[35] = k.g.R.Int63n(100)
|
||||
k.record[36] = k.g.StringFromListWeighted([]string{"snmp", "arp", "wlc", "cmms", "splunk", "bgp", "dhcp", "dns", "ftp", "http", "https",
|
||||
"imap", "ldap", "mgcp", "mqtt", "nntp", "ntp", "pop", "ptp", "onc/rpc", "rtp",
|
||||
"rtsp", "rip", "sip", "smtp", "ssh", "telnet", "tls/ssl"})
|
||||
k.record[37] = k.g.R.Int63n(100)
|
||||
k.record[38] = "unknown"
|
||||
k.record[39] = k.g.StringFromListWeighted([]string{"macbook.com", "imac.org", "google.com", "youtube.com", "fb.com", "csdn.net", "apple.com", "stackoverflow.com", "aws.com"})
|
||||
k.record[40] = k.g.R.Intn(2) == 0
|
||||
k.record[41] = k.g.StringFromListWeighted([]string{"offline", "inspector", "online", "SNMP", "nmap"})
|
||||
k.record[42] = ""
|
||||
k.record[43] = k.g.StringFromListWeighted([]string{"Embedded", "Desktop", "Server", "Workstation", "Handheld", "Hobbyist"})
|
||||
k.record[44] = "10.2.171.9"
|
||||
k.record[45] = k.g.R.Int63n(10)
|
||||
k.record[46] = k.g.StringFromListWeighted([]string{"unknown", "Firefox", "Chrome", "Safari", "Edge", "Internet Explorer", "Netscape Navigator", "Opera", "Iceweasel", "lynx", "IE6"})
|
||||
k.record[47] = os
|
||||
k.record[48] = k.g.StringFromListWeighted([]string{"offline", "inspector", "online", "SNMP", "nmap"})
|
||||
k.record[49] = company
|
||||
k.record[50] = company
|
||||
k.record[51] = k.g.R.Int63n(101)
|
||||
k.record[52] = "10.2.170.0/23"
|
||||
k.record[53] = k.g.StringFromListWeighted([]string{"Low", "Normal", "Zero", "Elevated", "High", "Critical"})
|
||||
k.record[54] = k.g.StringFromListWeighted([]string{"IoT", "Fixed", "Server"})
|
||||
k.record[55] = k.g.R.Int63n(101)
|
||||
k.record[56] = "unknown"
|
||||
k.record[57] = os
|
||||
k.record[58] = company
|
||||
k.record[59] = strconv.Itoa(k.g.R.Intn(10)) //"6"
|
||||
k.record[60] = k.g.R.Int63n(101)
|
||||
k.record[61] = baseTime.Unix() + k.g.R.Int63n(int64(durationSeconds))
|
||||
k.record[62] = baseTime.Unix() + k.g.R.Int63n(int64(durationSeconds))
|
||||
k.record[63] = "unknown"
|
||||
k.record[64] = baseTime.Unix() + k.g.R.Int63n(int64(durationSeconds))
|
||||
k.record[65] = k.g.StringFromListWeighted([]string{"Patient Tracking", "Doctor Tracking", "Equipment", "Droids You're Looking For"})
|
||||
k.record[66] = "000000000000000000000CC47AD" + strconv.Itoa(10000+k.g.R.Intn(89000))
|
||||
k.record[67] = k.g.StringFromListWeighted([]string{"medical", "technology", "retail", "manufacturing", "transportation", "mining", "hospitality"})
|
||||
k.record[68] = pql.NewDecimal(k.g.R.Int63n(10000), 4)
|
||||
k.record[69] = pql.NewDecimal(k.g.R.Int63n(10000), 4)
|
||||
k.record[70] = pql.NewDecimal(k.g.R.Int63n(10000), 4)
|
||||
k.record[71] = pql.NewDecimal(k.g.R.Int63n(10), 1)
|
||||
k.record[72] = pql.NewDecimal(k.g.R.Int63n(10), 1)
|
||||
k.record[73] = k.g.StringFromListWeighted([]string{"Windows CE", "Windows 2000", "Windows XP", "Windows 95", "Mac OS 8", "Solaris", "MS Bob", "Plan 9", "Windows ME Professional Plus Ultimate Turbo Edition"})
|
||||
k.record[74] = k.g.StringFromListWeighted([]string{"IoT", "Fixed", "Server"})
|
||||
k.record[75] = k.g.StringFromListWeighted([]string{"Yes", "No", "Unknown"})
|
||||
k.record[76] = "profiler"
|
||||
k.record[77] = baseTime.Unix() + k.g.R.Int63n(int64(durationSeconds))
|
||||
// semi-randomize mac address, cardinality 10k
|
||||
k.record[78].([]byte)[0] = 48 + byte(k.g.R.Intn(10))
|
||||
k.record[78].([]byte)[3] = 48 + byte(k.g.R.Intn(10))
|
||||
k.record[78].([]byte)[12] = 48 + byte(k.g.R.Intn(10))
|
||||
k.record[78].([]byte)[15] = 48 + byte(k.g.R.Intn(10))
|
||||
k.record[79] = "OuiClassifier"
|
||||
k.record[80] = k.g.StringFromListWeighted([]string{"Yes", "No", "Unknown"})
|
||||
k.record[81] = k.g.StringFromListWeighted([]string{"Patient Tracking", "Doctor Tracking", "Equipment", "Droids You're Looking For"})
|
||||
k.record[82] = k.g.StringFromListWeighted([]string{"iPhone", "iPad", "Mac-Macbook", "Mac-MacbookPro", "Mac-iMac",
|
||||
"Google Pixel 5", "Samsung Galaxy Note 20", "OnePlus 8 Pro",
|
||||
"Mac-MacMini", "Dell G3", "Microsoft Surface Pro", "Thinkpad",
|
||||
"HP Pavilion x360", "Dell Inspiron 7577", "HP Notebook", "HP Envy", "Asus Zenbook"})
|
||||
k.record[83] = k.g.R.Intn(2) == 0
|
||||
k.record[84] = k.g.R.Intn(2) == 0
|
||||
k.record[85] = baseTime.Unix() + k.g.R.Int63n(int64(durationSeconds))
|
||||
k.record[86] = ""
|
||||
k.record[87] = k.g.StringFromListWeighted([]string{"wireless", "wired", "bluetooth", "Qi", "unplugged"})
|
||||
k.record[88] = k.record[87]
|
||||
k.record[89] = k.g.StringFromListWeighted([]string{"not_protected", "protected", "double_coverage", "sacked"})
|
||||
k.record[90] = k.record[78]
|
||||
k.record[91] = k.g.StringFromListWeighted([]string{"Yes", "No", "Unknown"})
|
||||
k.record[92] = pql.NewDecimal(k.g.R.Int63n(10000), 2) // 98.52
|
||||
k.record[93] = baseTime.Unix() + k.g.R.Int63n(int64(durationSeconds))
|
||||
k.record[94] = "profiler"
|
||||
k.record[95] = baseTime.Unix() + k.g.R.Int63n(int64(durationSeconds))
|
||||
k.record[96] = k.g.R.Int63n(100) // 86
|
||||
k.record[97] = k.g.StringFromListWeighted([]string{"70_Medium", "50_Medium", "30_Medium", "10_Low", "90_High"})
|
||||
k.record[98] = k.g.R.Intn(4) != 0 // weight towards true
|
||||
k.record[99] = k.g.R.Intn(4) != 0 // weight towards true
|
||||
k.record[100] = k.g.StringFromListWeighted(k.objectIDs)
|
||||
k.record[101] = k.g.StringFromListWeighted([]string{"US", "AT", "AU", "BR", "CA", "CH", "CN", "DE",
|
||||
"DK", "EC", "EG", "FR", "GB", "IE", "IN", "IT", "JP", "MX", "NL",
|
||||
"PL", "SA", "SG", "TR", "UG", "ZM"})
|
||||
k.record[102] = k.g.R.Intn(4) == 0 // weight towards false
|
||||
// the first value, 15778080, is the timestamp value for 2000-01-01 at
|
||||
// the minute resolution. The second value used for the int range, 10999979,
|
||||
// is the duration in minutes between 2000-01-01 and 2020-11-30
|
||||
k.record[103] = 15778080 + k.g.R.Intn(10999979)
|
||||
|
||||
return k.record, nil
|
||||
}
|
||||
|
||||
func (k *PaloAltoSource) Schema() []idk.Field {
|
||||
return k.schema
|
||||
}
|
||||
|
||||
func (k *PaloAltoSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
91
idk/datagen/power_scenario.data.go
Normal file
91
idk/datagen/power_scenario.data.go
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
package datagen
|
||||
|
||||
var equip_ids = []int64{170, 6705, 8998, 15391, 15612, 15749, 21428, 24546, 35501, 36452, 37747, 39447, 39628, 40909,
|
||||
40946, 44663, 46444, 50279, 55768, 56147, 57967, 58219, 64503, 65026, 67258, 77028, 80852, 86930,
|
||||
88973, 92380, 94568, 98395, 103641, 105764, 110400, 111418, 125796, 133270, 134283, 136292, 155972, 156180,
|
||||
164954, 166068, 169949, 175969, 181630, 188810, 189121, 189617, 193660, 194865, 196097, 199576, 205504, 212633,
|
||||
218376, 219822, 221515, 223200, 225797, 228749, 229343, 230417, 233263, 234602, 240299, 246835, 256838, 257725,
|
||||
264436, 275532, 278068, 279703, 279957, 283968, 285540, 288279, 289852, 290472, 291381, 297128, 297533, 305117,
|
||||
307597, 308817, 308882, 311756, 312139, 315301, 317090, 323370, 327423, 329640, 338642, 344825, 356295, 362713,
|
||||
365282, 365828, 369949, 376736, 377181, 381091, 384663, 387357, 403746, 404146, 411349, 413003, 417046, 417612,
|
||||
418187, 418659, 422010, 422178, 422750, 426108, 427228, 434152, 438524, 438951, 450396, 452910, 459978, 460929,
|
||||
466267, 471529, 472060, 479897, 484332, 485665, 495865, 495963, 496292, 498195, 504239, 513424, 514008, 514727,
|
||||
514742, 518221, 520923, 534051, 535252, 544738, 547757, 548327, 553520, 554143, 554893, 556179, 558376, 559910,
|
||||
563382, 564769, 572059, 579761, 584028, 587525, 590231, 590471, 595030, 602503, 604893, 605096, 609045, 610622,
|
||||
615951, 616828, 625267, 625405, 627804, 628584, 645590, 649410, 654265, 655748, 658598, 661862, 662717, 669859,
|
||||
674301, 681368, 683068, 683443, 689674, 696713, 698216, 720728, 724265, 724466, 735765, 736057, 739503,
|
||||
741869, 747205, 748781, 752084, 754638, 756812, 759629, 776519, 778482, 780346, 784260, 785979, 787758, 792069,
|
||||
792128, 792668, 796483, 798071, 799332, 812631, 820151, 830777, 835937, 843404, 848141, 854779, 856719, 864956,
|
||||
865928, 868442, 877958, 879508, 880188, 888684, 891619, 897742, 898677, 902482, 903074, 903836, 904116, 906172,
|
||||
910796, 915183, 915622, 921337, 924944, 927204, 927524, 929183, 931793, 933467, 933744, 939942, 940675, 943807,
|
||||
952816, 961784, 973155, 983443, 990418, 991467, 996985, 1004994, 1006876, 1012563, 1018871, 1021258, 1024536, 1027764,
|
||||
1035367, 1042971, 1043520, 1045410, 1045909, 1046982, 1061690, 1063084, 1064184, 1064490, 1065741, 1066062, 1067023, 1067144,
|
||||
1078285, 1080073, 1080681, 1086342, 1099190, 1100930, 1102386, 1103079, 1106776, 1106794, 1113178, 1116643, 1117586, 1121897,
|
||||
1125231, 1150824, 1151526, 1156509, 1168490, 1169113, 1174033, 1175969, 1177760, 1181016, 1189303, 1194194, 1199435, 1199913,
|
||||
1205437, 1206431, 1207118, 1207588, 1213100, 1213774, 1216535, 1216671, 1219600, 1229116, 1230334, 1231457, 1240198, 1244987,
|
||||
1248359, 1248693, 1249288, 1249679, 1252185, 1252192, 1257594, 1258588, 1258837, 1259426, 1264695, 1270705, 1272356, 1275256,
|
||||
1276195, 1276355, 1286770, 1288261, 1289084, 1295310, 1297805, 1297867, 1300449, 1309905, 1311154, 1314725, 1316373, 1318509,
|
||||
1323662, 1323872, 1331279, 1341503, 1343486, 1355830, 1360519, 1368186, 1369393, 1377044, 1377355, 1379828, 1384774, 1385131,
|
||||
1390649, 1390752, 1392016, 1393066, 1397828, 1398403, 1402644, 1407699, 1408193, 1420595, 1421282, 1426560, 1428828, 1447146,
|
||||
1448096, 1454426, 1456991, 1459313, 1461845, 1464368, 1464775, 1469022, 1471894, 1477498, 1478516, 1479990, 1482303, 1487983,
|
||||
1489744, 1490877, 1495792, 1504572, 1511350, 1518125, 1518264, 1541773, 1545286, 1547050, 1548522, 1552861, 1564290, 1566414,
|
||||
1566431, 1577479, 1578614, 1580210, 1586196, 1592158, 1592803, 1609831, 1611124, 1613701, 1618272, 1623993, 1626796, 1630208,
|
||||
1630632, 1637402, 1640581, 1646833, 1649651, 1653907, 1661388, 1663303, 1677561, 1679624, 1684783, 1687466, 1689207, 1695593,
|
||||
1699313, 1703642, 1709621, 1711464, 1714225, 1714747, 1718371, 1719938, 1721880, 1722213, 1722378, 1726578, 1731519, 1732614,
|
||||
1737693, 1738711, 1740693, 1741217, 1741943, 1743220, 1744512, 1750249, 1753969, 1755937, 1756579, 1757821, 1765271, 1771636,
|
||||
1793789, 1798396, 1806097, 1809291, 1812137, 1824242, 1831634, 1837719, 1842044, 1842278, 1847888, 1850588, 1852475, 1852888,
|
||||
1856616, 1858738, 1862498, 1866149, 1867368, 1871306, 1871988, 1873287, 1874539, 1883931, 1885816, 1894204, 1894681, 1895614,
|
||||
1901123, 1903282, 1905423, 1909084, 1923727, 1930663, 1931956, 1932833, 1934820, 1937587, 1940728, 1941918, 1944165, 1946568,
|
||||
1947987, 1953801, 1953968, 1955186, 1956283, 1970037, 1970939, 1971283, 1971634, 1972028, 1974020, 1982146, 1984706, 1986285,
|
||||
1988839, 1994294, 1999537, 2001366, 2006323, 2019744, 2020244, 2021998, 2023253, 2024820, 2038708, 2043133, 2050415, 2054573,
|
||||
2055318, 2055641, 2060181, 2064735, 2066973, 2073400, 2076179, 2077647, 2082933, 2088378, 2091256, 2092254, 2095927, 2096728,
|
||||
2097256, 2098417, 2104590, 2107812, 2111894, 2113006, 2113172, 2115923, 2123194, 2124775, 2126841, 2127455, 2135914, 2138248,
|
||||
2138895, 2148831, 2152933, 2154423, 2157534, 2158772, 2165956, 2185189, 2188928, 2189922, 2191770, 2192335, 2197772, 2204293,
|
||||
2206929, 2207872, 2208524, 2212434, 2213670, 2216288, 2219663, 2222810, 2234302, 2236121, 2244474, 2245406, 2245618, 2247416,
|
||||
2256856, 2259832, 2266976, 2287105, 2292613, 2300919, 2308247, 2311442, 2317940, 2323171, 2328913, 2332308, 2338815, 2341978,
|
||||
2342680, 2343207, 2343258, 2347349, 2348530, 2356060, 2357599, 2366448, 2372157, 2373974, 2389857, 2397914, 2408614, 2409025,
|
||||
2413054, 2415388, 2418060, 2418287, 2431357, 2436957, 2437995, 2440135, 2448182, 2449312, 2450671, 2456767, 2458398, 2465373,
|
||||
2467950, 2471083, 2478256, 2479620, 2484480, 2485642, 2499422, 2501830, 2503617, 2507695, 2507924, 2514244, 2515993, 2517670,
|
||||
2521572, 2521758, 2521860, 2522431, 2523551, 2536359, 2542693, 2543890, 2544783, 2551368, 2553326, 2554384, 2555943, 2559870,
|
||||
2559978, 2560138, 2561731, 2562964, 2563032, 2566555, 2569629, 2572846, 2573293, 2577194, 2587236, 2591677, 2597597, 2598767,
|
||||
2603379, 2603936, 2610863, 2611226, 2612116, 2617492, 2623462, 2626900, 2627684, 2628938, 2629376, 2630167, 2630971, 2634686,
|
||||
2635285, 2638959, 2643208, 2650402, 2674968, 2675700, 2682692, 2686250, 2688569, 2698489, 2698756, 2701002, 2708988, 2711781,
|
||||
2721639, 2722444, 2728982, 2734144, 2736990, 2743465, 2746010, 2746122, 2750772, 2752595, 2760961, 2765330, 2766876, 2768977,
|
||||
2776873, 2780155, 2783415, 2785459, 2787728, 2789811, 2801913, 2810016, 2821657, 2825572, 2828022, 2835510, 2839090, 2841082,
|
||||
2845456, 2848734, 2853772, 2860625, 2862994, 2877543, 2877600, 2880718, 2882184, 2883714, 2887228, 2887901, 2889678, 2897354,
|
||||
2899312, 2904629, 2907542, 2916110, 2925284, 2927109, 2928100, 2929921, 2935813, 2947739, 2948218, 2949067, 2951839, 2952812,
|
||||
2958330, 2962633, 2969330, 2973762, 2974039, 2977915, 2981914, 2981974, 2993186, 3001478, 3009979, 3010958, 3017945, 3018092,
|
||||
3021502, 3023793, 3030820, 3048756, 3064636, 3066186, 3069757, 3074327, 3077454, 3082404, 3082597, 3082846, 3090844, 3092798,
|
||||
3097517, 3117732, 3120551, 3121731, 3123206, 3125144, 3129880, 3134051, 3138887, 3139269, 3146691, 3162623, 3167976, 3171680,
|
||||
3172092, 3174668, 3176520, 3178406, 3180494, 3181366, 3187286, 3192368, 3195066, 3207053, 3207135, 3213132, 3216378, 3222117,
|
||||
3235457, 3237770, 3238110, 3239629, 3254968, 3260403, 3271650, 3274503, 3278170, 3279265, 3281370, 3285205, 3287787, 3288274,
|
||||
3294659, 3297964, 3299741, 3304286, 3304555, 3306973, 3307749, 3318074, 3318193, 3321012, 3332069, 3333286, 3336583, 3357759,
|
||||
3366036, 3369940, 3373922, 3374020, 3380601, 3380834, 3385680, 3386484, 3388907, 3392291, 3392815, 3402052, 3410562, 3414439,
|
||||
3420520, 3423275, 3424701, 3425584, 3432973, 3433481, 3435264, 3439456, 3442491, 3447445, 3456075, 3458649, 3466648, 3470869,
|
||||
3475382, 3476606, 3476621, 3479464, 3487342, 3491444, 3493965, 3499913, 3504016, 3505277, 3510610, 3515067, 3521723, 3525368,
|
||||
3526858, 3533427, 3539037, 3543634, 3549370, 3554771, 3556188, 3556719, 3557106, 3558716, 3566081, 3568789, 3570470, 3577210,
|
||||
3577279, 3579450, 3580297, 3581380, 3584824, 3595719, 3596320, 3597828, 3599609, 3600356, 3610056, 3615088, 3615771, 3627152,
|
||||
3632238, 3633626, 3634000, 3634056, 3638155, 3641404, 3648305, 3650521, 3657979, 3663691, 3668525, 3671821, 3675056, 3690557,
|
||||
3691160, 3692120, 3699218, 3699407, 3700585, 3702982, 3707485, 3708256, 3708693, 3713871, 3714877, 3719320, 3722712, 3730939,
|
||||
3732068, 3736951, 3740221, 3742276, 3743132, 3744197, 3747296, 3752418, 3755690, 3760012, 3762328, 3762562, 3764389, 3766506,
|
||||
3770138, 3774772, 3777522, 3778852, 3782958, 3786682, 3790994, 3798517, 3806492, 3807916, 3812458, 3814356, 3817554, 3819970,
|
||||
3844018, 3847499, 3853682, 3858273, 3859005, 3859243, 3878788, 3886767, 3888247, 3894022, 3901758, 3905768, 3909665, 3913315,
|
||||
3916506, 3916508, 3921890, 3932260, 3933067, 3941306, 3945750, 3946479, 3948043, 3948982, 3953119, 3957406, 3963990, 3968883,
|
||||
3972848, 3973261, 3974825, 3976601, 3977770, 3977958, 3979475, 3984819, 3986007, 3992267, 3993561, 3994389, 4000917, 4001181,
|
||||
4014442, 4019390, 4022195, 4028877, 4029954, 4032587, 4037102, 4046403, 4048083, 4050063, 4056429, 4059822, 4064935, 4067753,
|
||||
4068991, 4074173, 4076051, 4078426, 4080941, 4086632, 4089832, 4093462, 4120531, 4126517, 4126989, 4128083, 4134362, 4135763,
|
||||
4136946, 4142396, 4153635, 4154133, 4155775, 4156353, 4159885, 4163437, 4173644, 4175669, 4175716, 4178444, 4182202, 4188460,
|
||||
4193633, 4195957, 4196111, 4200175, 4201044, 4203923, 4207626, 4211609, 4212894, 4219233, 4225031, 4227480, 4227943, 4235608,
|
||||
4237879, 4238186, 4242063, 4246550, 4254375, 4273300, 4273376, 4279125, 4288501, 4292289, 4303360, 4314767, 4319891, 4322988,
|
||||
4323330, 4331224, 4331940, 4332937, 4338874, 4351993, 4354255, 4367565, 4367809, 4379534, 4391845, 4401627, 4406904, 4409375,
|
||||
4416007, 4420605, 4422058, 4428228, 4428440, 4429111, 4430316, 4433444, 4442834, 4445844, 4450002, 4459236, 4462603, 4465108,
|
||||
4470150, 4470783, 4474314, 4477799, 4480164, 4485288, 4499664, 4502245, 4503453, 4515953, 4516662, 4518057, 4518154, 4523451,
|
||||
4525325, 4528875, 4534950, 4535080, 4536380, 4537342, 4541059, 4542658, 4557608, 4561213, 4564417, 4570249, 4580957, 4581770,
|
||||
4585987, 4588300, 4601158, 4602739, 4602774, 4604514, 4607159, 4610051, 4616441, 4619168, 4620611, 4623445, 4626226, 4628480,
|
||||
4637825, 4644310, 4647256, 4660825, 4669516, 4670709, 4677429, 4677486, 4678820, 4679973, 4690245, 4690275, 4695766, 4699852,
|
||||
4702631, 4704632, 4712369, 4715789, 4719854, 4722861, 4722918, 4725870, 4729612, 4734176, 4741068, 4743653, 4757745, 4760132,
|
||||
4763429, 4763479, 4771243, 4772347, 4779714, 4779717, 4784801, 4790225, 4790556, 4793134, 4798198, 4799146, 4799455, 4800860,
|
||||
4811057, 4821071, 4823592, 4834250, 4838501, 4841596, 4842353, 4844741, 4851381, 4855034, 4857285, 4863730, 4864267, 4866872,
|
||||
4869466, 4869896, 4872492, 4874296, 4877102, 4883864, 4888686, 4891102, 4891984, 4913401, 4913731, 4922248, 4925341, 4927279,
|
||||
4929436, 4938404, 4955445, 4958196, 4963007, 4963658, 4964288, 4968824, 4970773, 4982366, 4991538, 4991817, 4993291, 4995733,
|
||||
4996460, 4999789}
|
||||
147
idk/datagen/power_scenario_1.go
Normal file
147
idk/datagen/power_scenario_1.go
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Power1 implements interface.
|
||||
var _ Sourcer = (*Power1)(nil)
|
||||
|
||||
// Power1 implements Sourcer.
|
||||
type Power1 struct {
|
||||
startEnds []startEnd
|
||||
|
||||
mu sync.Mutex
|
||||
sourceCall int
|
||||
}
|
||||
|
||||
// NewPower1 returns a new instance of Power1.
|
||||
func NewPower1(cfg SourceGeneratorConfig) Sourcer {
|
||||
start := cfg.StartFrom
|
||||
end := cfg.EndAt
|
||||
if start == 0 {
|
||||
start = 100000000
|
||||
if end == 0 {
|
||||
end = start + 630
|
||||
} else {
|
||||
end += 100000000
|
||||
}
|
||||
}
|
||||
// We need to adjust end to be exclusive.
|
||||
end++
|
||||
|
||||
customCfg := SourceGeneratorConfig{
|
||||
StartFrom: start,
|
||||
EndAt: end,
|
||||
Concurrency: cfg.Concurrency,
|
||||
}
|
||||
|
||||
_, ses, _ := startEnds(customCfg)
|
||||
|
||||
return &Power1{
|
||||
startEnds: ses,
|
||||
}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (p *Power1) Source(cfg SourceConfig) idk.Source {
|
||||
p.mu.Lock()
|
||||
i := p.sourceCall
|
||||
p.sourceCall++
|
||||
p.mu.Unlock()
|
||||
src := &Power1Source{
|
||||
// We're goind to ignore the passed in start/end,
|
||||
// and instead used that calculated by NewPower1.
|
||||
cur: p.startEnds[i].start,
|
||||
endAt: p.startEnds[i].end,
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "data_type"}, // 1
|
||||
idk.IntField{NameVal: "equip_id"}, // 2
|
||||
idk.StringField{NameVal: "group"}, // 3
|
||||
idk.IntField{NameVal: "int_value"}, // 4
|
||||
idk.StringField{NameVal: "string_value"}, // 5
|
||||
idk.IntField{NameVal: "timestamp"}, // 6
|
||||
idk.StringField{NameVal: "type"}, // 7
|
||||
},
|
||||
}
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (p *Power1) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (p *Power1) DefaultEndAt() uint64 {
|
||||
return 100000630
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (p *Power1) Info() string {
|
||||
return "Generates data representative of Power Analytics operations (#1)."
|
||||
}
|
||||
|
||||
// Ensure Power1Source implements interface.
|
||||
var _ idk.Source = (*Power1Source)(nil)
|
||||
|
||||
type Power1Source struct {
|
||||
Log logger.Logger
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
|
||||
record record
|
||||
}
|
||||
|
||||
func (s *Power1Source) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.cur
|
||||
|
||||
// timestamp
|
||||
s.record[6] = int(509100000 + (10 * s.cur))
|
||||
|
||||
// type
|
||||
s.record[7] = "Amperage"
|
||||
|
||||
// group
|
||||
s.record[3] = "single"
|
||||
|
||||
// equip_id
|
||||
s.record[2] = 712278 // breaker at a retail site
|
||||
|
||||
// data type and string or int value
|
||||
s.record[1] = "int"
|
||||
s.record[4] = 30
|
||||
s.record[5] = nil
|
||||
|
||||
s.cur++
|
||||
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *Power1Source) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *Power1Source) Close() error {
|
||||
return nil
|
||||
}
|
||||
125
idk/datagen/power_scenario_1_2.go
Normal file
125
idk/datagen/power_scenario_1_2.go
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Power1_2 implements interface.
|
||||
var _ Sourcer = (*Power1_2)(nil)
|
||||
|
||||
// Power1_2 implements Sourcer.
|
||||
type Power1_2 struct{}
|
||||
|
||||
// NewPower1_2 returns a new instance of Power1_2.
|
||||
func NewPower1_2(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Power1_2{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (p *Power1_2) Source(cfg SourceConfig) idk.Source {
|
||||
src := &Power1_2Source{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "data_type"}, // 1
|
||||
idk.IntField{NameVal: "equip_id"}, // 2
|
||||
idk.StringField{NameVal: "group"}, // 3
|
||||
idk.IntField{NameVal: "int_value"}, // 4
|
||||
idk.StringField{NameVal: "string_value"}, // 5
|
||||
idk.IntField{NameVal: "timestamp"}, // 6
|
||||
idk.StringField{NameVal: "type"}, // 7
|
||||
},
|
||||
}
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (p *Power1_2) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (p *Power1_2) DefaultEndAt() uint64 {
|
||||
return 3*uint64(len(equip_ids)) - 1
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (p *Power1_2) Info() string {
|
||||
return "Generates data representative of Power Analytics operations (#2)."
|
||||
}
|
||||
|
||||
// Ensure Power1_2Source implements interface.
|
||||
var _ idk.Source = (*Power1_2Source)(nil)
|
||||
|
||||
type Power1_2Source struct {
|
||||
Log logger.Logger
|
||||
|
||||
rand *rand.Rand
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
|
||||
record record
|
||||
}
|
||||
|
||||
func (s *Power1_2Source) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = 101000000 + s.cur
|
||||
|
||||
// timestamp
|
||||
//s.record[6] = timestamp
|
||||
|
||||
// type
|
||||
s.record[7] = "Amperage"
|
||||
|
||||
// group
|
||||
s.record[3] = "free"
|
||||
|
||||
// equip_id
|
||||
s.record[2] = equip_ids[s.cur%uint64(len(equip_ids))]
|
||||
|
||||
// timestamp
|
||||
s.record[6] = int(1509100001) + 3600*int(s.cur/uint64(len(equip_ids)))
|
||||
|
||||
// data type and string or int value
|
||||
s.record[1] = "int"
|
||||
s.record[4] = s.rand.Intn(10000) + 50
|
||||
s.record[5] = nil
|
||||
|
||||
s.cur++
|
||||
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *Power1_2Source) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *Power1_2Source) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*Power1_2Source)(nil)
|
||||
|
||||
func (s *Power1_2Source) Close() error {
|
||||
return nil
|
||||
}
|
||||
28
idk/datagen/shared.go
Normal file
28
idk/datagen/shared.go
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
package datagen
|
||||
|
||||
import "time"
|
||||
|
||||
var customers = []string{"CVS", "McDonald's", "Walmart", "Visa", "Sprint", "Comcast"}
|
||||
var inStorePayments = []string{"Credit Card", "Debit Card", "Cash", "ACH", "Return"}
|
||||
var ecomPayments = []string{"Credit Card", "Debit Card", "ACH", "Return"}
|
||||
var tsTypes = []string{"Voltage", "VoltageRMS", "Amperage", "Resistance", "Frequency", "Message"}
|
||||
|
||||
var startDayInt int64
|
||||
var todayInt int64
|
||||
|
||||
var startDate = time.Date(2015, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
var today = time.Date(2019, time.November, 19, 0, 0, 0, 0, time.UTC)
|
||||
var currentDur = today.Sub(startDate)
|
||||
|
||||
var daysSinceStart = int(currentDur / (time.Hour * 24))
|
||||
|
||||
func init() {
|
||||
epoch := time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
start := time.Date(today.Year()-10, today.Month(), today.Day(), today.Hour(), today.Minute(), today.Second(), today.Nanosecond(), today.Location())
|
||||
startDayInt = int64(start.Sub(epoch) / (time.Hour * 24))
|
||||
todayInt = int64(today.Sub(epoch) / (time.Hour * 24))
|
||||
}
|
||||
|
||||
func intptr(i int64) *int64 {
|
||||
return &i
|
||||
}
|
||||
37
idk/datagen/sites.data.go
Normal file
37
idk/datagen/sites.data.go
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
package datagen
|
||||
|
||||
var maintainerDomains = []string{"@datafoundry.com", "@digitalrealty.com", "@globalswitch.us", "@ironmountain.com", "@cyrusone.com", "@chinatelecom-h.com", "@equinix.com", "@chinaunicom.com.hk", "@telehouse.com", "@ntt.com"}
|
||||
|
||||
var serviceProviders = []string{"Data Foundry", "Digital Realty Trust", "Global Switch", "Iron Mountain", "CyrusOne", "China Telecom", "Equinix", "China Unicom", "Telehouse", "NTT Communications"}
|
||||
|
||||
var lastNames = []string{"smith", "johnson", "williams", "jones", "brown", "davis", "miller", "wilson",
|
||||
"moore", "taylor", "anderson", "thomas", "jackson", "white", "harris", "martin", "thompson",
|
||||
"garcia", "martinez", "robinson", "clark", "rodriguez", "lewis", "lee", "walker", "hall",
|
||||
"allen", "young", "hernandez", "king", "wright", "lopez", "hill", "scott", "green",
|
||||
"adams", "baker", "gonzalez", "nelson", "carter", "mitchell", "perez", "roberts", "turner",
|
||||
"phillips", "campbell", "parker", "evans", "edwards", "collins", "stewart", "sanchez", "morris",
|
||||
"rogers", "reed", "cook", "morgan", "bell", "murphy", "bailey", "rivera", "cooper",
|
||||
"richardson", "cox", "howard", "ward", "torres", "peterson", "gray", "ramirez", "james",
|
||||
"watson", "brooks", "kelly", "sanders", "price", "bennett", "wood", "barnes", "ross",
|
||||
"henderson", "coleman", "jenkins", "perry", "powell", "long", "patterson", "hughes", "flores",
|
||||
"washington", "butler", "simmons", "foster", "gonzales", "bryant", "alexander", "russell", "griffin",
|
||||
"diaz", "hayes", "myers", "ford", "hamilton", "graham", "sullivan", "wallace", "woods",
|
||||
"cole", "west", "jordan", "owens", "reynolds", "fisher", "ellis", "harrison", "gibson",
|
||||
"mcdonald", "cruz", "marshall", "ortiz", "gomez", "murray", "freeman", "wells", "webb",
|
||||
"simpson", "stevens", "tucker", "porter", "hunter", "hicks", "crawford", "henry", "boyd",
|
||||
"mason", "morales", "kennedy", "warren", "dixon", "ramos", "reyes", "burns", "gordon",
|
||||
"shaw", "holmes", "rice", "robertson", "hunt", "black", "daniels", "palmer", "mills",
|
||||
"nichols", "grant", "knight", "ferguson", "rose", "stone", "hawkins", "dunn", "perkins",
|
||||
"hudson", "spencer", "gardner", "stephens", "payne", "pierce", "berry", "matthews", "arnold",
|
||||
"wagner", "willis", "ray", "watkins", "olson", "carroll", "duncan", "snyder", "hart",
|
||||
"cunningham", "bradley", "lane", "andrews", "ruiz", "harper", "fox", "riley", "armstrong",
|
||||
"carpenter", "weaver", "greene", "lawrence", "elliott", "chavez", "sims", "austin", "peters",
|
||||
"kelley", "franklin", "lawson", "fields", "gutierrez", "ryan", "schmidt", "carr", "vasquez",
|
||||
"castillo", "wheeler", "chapman", "oliver", "montgomery", "richards", "williamson", "johnston", "banks",
|
||||
"meyer", "bishop", "mccoy", "howell", "alvarez", "morrison", "hansen", "fernandez", "garza",
|
||||
"harvey", "little", "burton", "stanley", "nguyen", "george", "jacobs", "reid", "kim",
|
||||
"fuller", "lynch", "dean", "gilbert", "garrett", "romero", "welch", "larson", "frazier",
|
||||
"burke", "hanson", "day", "mendoza", "moreno", "bowman", "medina", "fowler", "brewer",
|
||||
"hoffman", "carlson", "silva", "pearson", "holland", "douglas", "fleming", "jensen", "vargas",
|
||||
"byrd", "davidson", "hopkins", "may", "terry", "herrera", "wade", "soto", "walters",
|
||||
"curtis", "neal", "caldwell", "lowe"}
|
||||
211
idk/datagen/sites.go
Normal file
211
idk/datagen/sites.go
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Site implements interface.
|
||||
var _ Sourcer = (*Site)(nil)
|
||||
|
||||
// Site implements Sourcer.
|
||||
type Site struct{}
|
||||
|
||||
// NewSite returns a new instance of Site.
|
||||
func NewSite(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Site{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (s *Site) Source(cfg SourceConfig) idk.Source {
|
||||
src := &SiteSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.DecimalField{NameVal: "latitude", Scale: 4}, // 1
|
||||
idk.DecimalField{NameVal: "longitude", Scale: 4}, // 2
|
||||
idk.StringField{NameVal: "maintainer"}, // 3
|
||||
idk.StringField{NameVal: "classification"}, // 4
|
||||
idk.StringField{NameVal: "type"}, // 5
|
||||
idk.StringField{NameVal: "service_provider"}, // 6
|
||||
idk.StringField{NameVal: "region"}, // 7
|
||||
idk.DateIntField{NameVal: "last_inspection"}, // 8
|
||||
},
|
||||
}
|
||||
|
||||
src.providerZipf = rand.NewZipf(src.rand, 1.03, 3, uint64(len(serviceProviders))-1)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[7] = int(1420070400)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (s *Site) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (s *Site) DefaultEndAt() uint64 {
|
||||
return 4688000 - 1
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (s *Site) Info() string {
|
||||
return "Generates site data (customer, size, location) - includes a foreign key for network_ts, power_ts, transaction_ts."
|
||||
|
||||
}
|
||||
|
||||
// Ensure SiteSource implements interface.
|
||||
var _ idk.Source = (*SiteSource)(nil)
|
||||
|
||||
type SiteSource struct {
|
||||
Log logger.Logger
|
||||
|
||||
rand *rand.Rand
|
||||
providerZipf *rand.Zipf
|
||||
|
||||
schema []idk.Field
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
record record
|
||||
}
|
||||
|
||||
func (s *SiteSource) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.cur
|
||||
|
||||
// lat/lon
|
||||
longitude, latitude := s.location()
|
||||
s.record[1] = latitude
|
||||
s.record[2] = longitude
|
||||
|
||||
// service provider
|
||||
var domain string
|
||||
s.record[6], domain = s.serviceProviderDomain()
|
||||
|
||||
// maintainer
|
||||
s.record[3] = s.emailName() + domain
|
||||
|
||||
// classification
|
||||
s.record[4] = s.classification()
|
||||
|
||||
// type
|
||||
s.record[5] = s.siteType(s.cur)
|
||||
if s.record[5].(string) == "" {
|
||||
return nil, errors.New("exhausted sites data set")
|
||||
}
|
||||
|
||||
// region
|
||||
s.record[7] = region(latitude, longitude)
|
||||
|
||||
// last_inspection
|
||||
s.record[8] = startDayInt + s.rand.Int63n(365*10-200)
|
||||
|
||||
s.cur++
|
||||
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *SiteSource) classification() string {
|
||||
num := s.rand.Intn(100)
|
||||
switch {
|
||||
case num < 3:
|
||||
return "Tier 1"
|
||||
case num < 20:
|
||||
return "Tier 2"
|
||||
case num < 40:
|
||||
return "Tier 3"
|
||||
case num < 75:
|
||||
return "Tier 4"
|
||||
default:
|
||||
return "Tier 5"
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SiteSource) serviceProviderDomain() (string, string) {
|
||||
idx := s.providerZipf.Uint64()
|
||||
return serviceProviders[idx], maintainerDomains[idx]
|
||||
}
|
||||
|
||||
func (s *SiteSource) emailName() string {
|
||||
idx := s.rand.Intn(len(lastNames))
|
||||
lastName := lastNames[idx]
|
||||
initial := uint8(s.rand.Intn(26) + 97)
|
||||
return string([]byte{initial, '.'}) + lastName
|
||||
}
|
||||
|
||||
func (s *SiteSource) siteType(id uint64) string {
|
||||
switch {
|
||||
case id < 1000:
|
||||
return "MSC" // 1k
|
||||
case id < 8000:
|
||||
return "Headend" // 7k
|
||||
case id < 33000:
|
||||
return "Central Office" // 25k
|
||||
case id < 88000:
|
||||
return "Substation" // 55k
|
||||
case id < 488000:
|
||||
return "Retail" // 400k
|
||||
case id < 888000:
|
||||
return "Cell Site" // 400k
|
||||
case id < 1688000:
|
||||
return "NOC" // 800k
|
||||
case id < 4688000:
|
||||
return "Data Center" // 3M
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SiteSource) location() (lon float64, lat float64) {
|
||||
loc := locations[s.rand.Intn(len(locations))]
|
||||
off1, off2 := (s.rand.Float64()-0.5)/50, (s.rand.Float64()-0.5)/50
|
||||
return loc.lon + off1, loc.lat + off2
|
||||
}
|
||||
|
||||
func region(latitude, longitude float64) string {
|
||||
if longitude < 80 {
|
||||
return "East"
|
||||
}
|
||||
if longitude > 105 {
|
||||
return "West"
|
||||
}
|
||||
if latitude < 39 {
|
||||
return "South"
|
||||
}
|
||||
return "North"
|
||||
}
|
||||
|
||||
func (s *SiteSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *SiteSource) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*SiteSource)(nil)
|
||||
|
||||
func (s *SiteSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
124
idk/datagen/sizing.go
Normal file
124
idk/datagen/sizing.go
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
pilosaclient "github.com/molecula/featurebase/v3/client"
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Sizing implements interface.
|
||||
var _ Sourcer = (*Sizing)(nil)
|
||||
|
||||
// Sizing implements Sourcer.
|
||||
type Sizing struct{}
|
||||
|
||||
// NewSizing returns a new instance of Sizing.
|
||||
func NewSizing(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Sizing{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (s *Sizing) Source(cfg SourceConfig) idk.Source {
|
||||
src := &SizingSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"},
|
||||
idk.IntField{NameVal: "eightbit_random"},
|
||||
idk.IntField{NameVal: "sixteenbit_random"},
|
||||
idk.IntField{NameVal: "thirtytwobit_random"},
|
||||
idk.IntField{NameVal: "sixtythreebit_random"},
|
||||
idk.IntField{NameVal: "eightbit_zipf"},
|
||||
idk.IntField{NameVal: "sixteenbit_zipf"},
|
||||
idk.IntField{NameVal: "thirtytwobit_zipf"},
|
||||
idk.IntField{NameVal: "sixtythreebit_zipf"},
|
||||
},
|
||||
}
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.zipf8 = rand.NewZipf(src.rand, 1.01, 4, 255)
|
||||
src.zipf16 = rand.NewZipf(src.rand, 1.01, 4, 65535)
|
||||
src.zipf32 = rand.NewZipf(src.rand, 1.01, 4, 4294967295)
|
||||
src.zipf63 = rand.NewZipf(src.rand, 1.01, 4, 9223372036854775807)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (s *Sizing) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (s *Sizing) DefaultEndAt() uint64 {
|
||||
return pilosaclient.DefaultShardWidth - 1 // one shard width
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (s *Sizing) Info() string {
|
||||
return "Generates data for estimating index size relative to original data size."
|
||||
}
|
||||
|
||||
// Ensure SizingSource implements interface.
|
||||
var _ idk.Source = (*SizingSource)(nil)
|
||||
|
||||
// SizingSource is an idk.Source meant to generate data which is
|
||||
// helpful in determining the on-disk footprint of different types of
|
||||
// data which can be extrapolated to help estimate necessary
|
||||
// infrastructure size for various data. Typically one shard width of
|
||||
// data is generated.
|
||||
type SizingSource struct {
|
||||
Log logger.Logger
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
rand *rand.Rand
|
||||
zipf8 *rand.Zipf
|
||||
zipf16 *rand.Zipf
|
||||
zipf32 *rand.Zipf
|
||||
zipf63 *rand.Zipf
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
}
|
||||
|
||||
func (s *SizingSource) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.cur
|
||||
s.record[1] = s.rand.Intn(255)
|
||||
s.record[2] = s.rand.Intn(65535)
|
||||
s.record[3] = s.rand.Intn(4294967295)
|
||||
s.record[4] = s.rand.Intn(9223372036854775807)
|
||||
s.record[5] = int64(s.zipf8.Uint64())
|
||||
s.record[6] = int64(s.zipf16.Uint64())
|
||||
s.record[7] = int64(s.zipf32.Uint64())
|
||||
s.record[8] = int64(s.zipf63.Uint64())
|
||||
s.cur++
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *SizingSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *SizingSource) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*SizingSource)(nil)
|
||||
|
||||
func (s *SizingSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
218
idk/datagen/skills.data.go
Normal file
218
idk/datagen/skills.data.go
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
package datagen
|
||||
|
||||
var skills = []string{
|
||||
"Data Entry",
|
||||
"Answering Phones",
|
||||
"Billing",
|
||||
"Bookkeeping (Excel, Turbotax)",
|
||||
"Scheduling",
|
||||
"Ms Office",
|
||||
"Office Equipment",
|
||||
"Quickbooks",
|
||||
"Shipping",
|
||||
"Welcoming Visitors",
|
||||
"Salesforce",
|
||||
"Calendar Management",
|
||||
"Organization",
|
||||
"Time Management",
|
||||
"Problem Solving",
|
||||
"Attention To Detail",
|
||||
"Adaptability",
|
||||
"Written Communication",
|
||||
"Email Platforms (E.G., Gmail And iCloud Mail)",
|
||||
"Word Processing Software (E.G., Google Docs)",
|
||||
"Spreadsheet Software",
|
||||
"Databases",
|
||||
"Digital Calendars",
|
||||
"Prioritization",
|
||||
"Product Knowledge",
|
||||
"Lead Qualification",
|
||||
"Lead Prospecting",
|
||||
"Customer Needs Analysis",
|
||||
"Referral Marketing",
|
||||
"Contract Negotiation",
|
||||
"Self Motivation",
|
||||
"Increasing Customer Lifetime Value (Clv)",
|
||||
"Reducing Customer Acquisition Cost (Cac)",
|
||||
"CRM Software (Salesforce, Hubspot, Zoho, Freshsales)",
|
||||
"POS Skills",
|
||||
"Cashier Skills",
|
||||
"Problem Solving",
|
||||
"Patience",
|
||||
"Knowledge Of Products And Services",
|
||||
"Adaptability",
|
||||
"Customer Service",
|
||||
"Communication",
|
||||
"Math Skills",
|
||||
"Organization",
|
||||
"Time Management",
|
||||
"Negotiation",
|
||||
"Persuasion",
|
||||
"Patience",
|
||||
"Diplomacy",
|
||||
"Cold Calling",
|
||||
"Customer Relationship Management (CRM)",
|
||||
"Lead Generation",
|
||||
"Buyer-responsive Selling",
|
||||
"Buyer Engagement",
|
||||
"Teamwork",
|
||||
"Patient Assessment",
|
||||
"Taking Vital Signs",
|
||||
"Patient Care",
|
||||
"Recording Patient Medical History",
|
||||
"Wound Dressing And Care",
|
||||
"Urgent And Emergency Care",
|
||||
"Record-keeping",
|
||||
"Patient Education",
|
||||
"NIH Stroke Scale Patient Assessment",
|
||||
"Electronic Medical Record (EMR)",
|
||||
"Medicine Administration",
|
||||
"Blood Pressure Monitoring",
|
||||
"Phlebotomy",
|
||||
"Rehabilitation Therapy",
|
||||
"Hygiene Assistance",
|
||||
"Use Of X-ray, MRI, Cat Scans",
|
||||
"Meditech",
|
||||
"Glucose Checks",
|
||||
"Electronic Heart Record (EHR)",
|
||||
"Time Management",
|
||||
"Prioritization",
|
||||
"Decision-Making Ability",
|
||||
"Critical Thinking",
|
||||
"Leadership",
|
||||
"Thorough Understanding Of HIPPA And Privacy Policies",
|
||||
"Ability To Take And Record Vital Signs",
|
||||
"CPR And First-Aid Certifications",
|
||||
"Perform And Evaluate Diagnostic Tests",
|
||||
"Maintain Patient Charts",
|
||||
"Programming Languages",
|
||||
"Web Development",
|
||||
"Data Structures",
|
||||
"Open Source Experience",
|
||||
"Security",
|
||||
"Machine Learning",
|
||||
"Debugging",
|
||||
"UX/UI",
|
||||
"Front-End & Back-End Development",
|
||||
"Cloud Management",
|
||||
"Agile Development",
|
||||
"HTML/CSS",
|
||||
"CSS Preprocessors",
|
||||
"Javascript",
|
||||
"Wordpress",
|
||||
"Graphic User Interfaces (GUI)",
|
||||
"Git/Version Control (Github, gitlab)",
|
||||
"Search Engine Optimization (SEO)",
|
||||
"Application Programming Interface (API)",
|
||||
"Adobe Photoshop, InDesign",
|
||||
"Content Management Systems (CMS)",
|
||||
"Testing/Debugging",
|
||||
"Responsive Design Principles",
|
||||
"Mobile and Web Development (e.g. iOS, Android)",
|
||||
"STEM Skills",
|
||||
"CAD",
|
||||
"Design",
|
||||
"Prototyping",
|
||||
"Testing",
|
||||
"Troubleshooting",
|
||||
"Project Launch",
|
||||
"Lean Manufacturing",
|
||||
"Workflow Development",
|
||||
"Computer Skills",
|
||||
"SolidWorks",
|
||||
"Budgeting",
|
||||
"Technical Report Writing",
|
||||
"Programming Languages HTML, CSS, CRM tools",
|
||||
"SEO (SEMRush, WordPress, and Ahrefs)",
|
||||
"SEM (i.e., Google Adwords)",
|
||||
"PPC",
|
||||
"CRO and A/B Testing",
|
||||
"Social Media Marketing and Paid Social Media Advertising",
|
||||
"Sales Funnel Management",
|
||||
"CMS Tools (WordPress, Weebly)",
|
||||
"Graphic Design Skills (Adobe Creative Suite)",
|
||||
"Email Marketing (MailChimp, Constant Contact)",
|
||||
"Email Automation",
|
||||
"Data Visualization",
|
||||
"CPC",
|
||||
"Typography",
|
||||
"Print Design",
|
||||
"Photography and Branding",
|
||||
"Communication",
|
||||
"Creativity",
|
||||
"Data Analytics (Google Analytics )",
|
||||
"Web Analytics",
|
||||
"Adaptability",
|
||||
"Organized",
|
||||
"Email Writing",
|
||||
"Google Adwords",
|
||||
"Social Media And Mobile Marketing",
|
||||
"Paid Social Media Advertisements",
|
||||
"Consumer Behavior Drivers",
|
||||
"Brand Management",
|
||||
"B2b Marketing",
|
||||
"Writing Advertising Copy",
|
||||
"Soliciting Feedback From Customers",
|
||||
"Cutting Costs",
|
||||
"Leadership skills",
|
||||
"Task Delegation",
|
||||
"Strategic Management",
|
||||
"Negotiation",
|
||||
"Planning",
|
||||
"Proposal writing",
|
||||
"Problem-solving",
|
||||
"Innovation",
|
||||
"Charisma",
|
||||
"Problem solving",
|
||||
"Personable",
|
||||
"Website Management",
|
||||
"Social Media Outreach",
|
||||
"Video Production",
|
||||
"Data Visualization",
|
||||
"Campaign Management",
|
||||
"Photo Editing",
|
||||
"Editing",
|
||||
"Typography",
|
||||
"Logo Creation",
|
||||
"Digital Printing",
|
||||
"Interactive Media Design",
|
||||
"Color Sense & Theory",
|
||||
"Ad Design",
|
||||
"Social Media Publishing",
|
||||
"UX Design",
|
||||
"Storytelling",
|
||||
"Financial Analysis",
|
||||
"Consumer Research",
|
||||
"Statistical Analysis",
|
||||
"Drawing",
|
||||
"Design",
|
||||
"Videography",
|
||||
"Technical Writing",
|
||||
"Mobile and Web Development (e.g. iOS, Android)",
|
||||
"Agile Project Management (Kanban)",
|
||||
"Managing Cross-Functional Teams",
|
||||
"Scrum Management",
|
||||
"Performance Tracking",
|
||||
"Financial Modelling",
|
||||
"Ideation Leadership",
|
||||
"Feature Definition",
|
||||
"Forecasting",
|
||||
"Profit and Loss",
|
||||
"Scope Management",
|
||||
"Project Lifecycle Management",
|
||||
"Meeting Facilitation",
|
||||
"Leadership",
|
||||
"Communication",
|
||||
"Critical Thinking",
|
||||
"Time Management",
|
||||
"Mentoring",
|
||||
"Risk Management",
|
||||
"Negotiation",
|
||||
"Cost Management",
|
||||
"Data Analysis",
|
||||
"Collaborative",
|
||||
"Collaborative Programs (Slack, WhatsApp, Dropbox)",
|
||||
"Wordpress",
|
||||
"Adobe Creative Suite",
|
||||
"Performance Tracking",
|
||||
}
|
||||
94
idk/datagen/stringpk.go
Normal file
94
idk/datagen/stringpk.go
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
)
|
||||
|
||||
// Ensure StringPK implements interface.
|
||||
var _ Sourcer = (*StringPK)(nil)
|
||||
|
||||
// StringPK implements Sourcer.
|
||||
type StringPK struct{}
|
||||
|
||||
// NewStringPK returns a new instance of StringPK.
|
||||
func NewStringPK(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &StringPK{}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (s *StringPK) Source(cfg SourceConfig) idk.Source {
|
||||
sps := &StringPKSource{
|
||||
record: make(record, 2),
|
||||
n: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
}
|
||||
|
||||
sps.record[0] = make([]byte, 12)
|
||||
sps.g = gen.New(gen.OptGenSeed(cfg.seed))
|
||||
|
||||
return sps
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (s *StringPK) PrimaryKeyFields() []string {
|
||||
return []string{"pk"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (s *StringPK) DefaultEndAt() uint64 {
|
||||
return 10000000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (s *StringPK) Info() string {
|
||||
return "TODO"
|
||||
}
|
||||
|
||||
// Ensure StringPKSource implements interface.
|
||||
var _ idk.Source = (*StringPKSource)(nil)
|
||||
|
||||
type StringPKSource struct {
|
||||
g *gen.Gen
|
||||
|
||||
endAt uint64
|
||||
n uint64
|
||||
|
||||
record record
|
||||
}
|
||||
|
||||
func (s *StringPKSource) Record() (idk.Record, error) {
|
||||
if s.n >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.g.AlphaUpper(s.record[0].([]byte))
|
||||
s.record[1] = s.g.R.Intn(10)
|
||||
s.n++
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *StringPKSource) Schema() []idk.Field {
|
||||
return []idk.Field{
|
||||
idk.StringField{NameVal: "pk"},
|
||||
idk.IDField{NameVal: "field1"},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StringPKSource) Seed(seed int64) {
|
||||
s.g.R.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*StringPKSource)(nil)
|
||||
|
||||
func (s *StringPKSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
119
idk/datagen/testdata/custom.yaml
vendored
Normal file
119
idk/datagen/testdata/custom.yaml
vendored
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
fields:
|
||||
- name: "an_id"
|
||||
type: "uint" # (default IDField (non-mutex))
|
||||
distribution: "sequential"
|
||||
min: 0
|
||||
max: 1000000
|
||||
repeat: false # if false, data generation stops when we hit >= max. only available with sequential
|
||||
step: 1
|
||||
- name: "an_int"
|
||||
type: "int" # (default IntField)
|
||||
distribution: "uniform" # uniform or zipfian
|
||||
min: 0
|
||||
max: 5
|
||||
null_chance: 0.01
|
||||
- name: "a_zipf_int"
|
||||
type: "int" # (default IntField)
|
||||
distribution: "zipfian"
|
||||
min: 0
|
||||
max: 100
|
||||
s: 1.1
|
||||
v: 5.1
|
||||
- name: "a_string_from_file"
|
||||
type: "string" # (default StringField (non-mutex))
|
||||
source_file: "testdata/file_with_line_delimited_values" # this will be used if not empty
|
||||
distribution: "zipfian" # uniform or zipfian
|
||||
s: 1.1 # s and v are used for zipfian
|
||||
v: 5.1
|
||||
- name: "a_random_string"
|
||||
type: "string" # (default StringField (non-mutex))
|
||||
generator_type: "random-string" # used to generate random strings rather than pulling from known set
|
||||
min_len: 8
|
||||
max_len: 12
|
||||
charset: "AB" # set of possible characters to pull from when generating random string
|
||||
- name: "a_string_set"
|
||||
type: "string-set" # (default StringArrayField)
|
||||
generator_type: "random-string" # used to generate random strings rather than pulling from known. "distribution" is ignored.
|
||||
min_len: 4
|
||||
max_len: 4
|
||||
charset: "0123456789ABCDEF"
|
||||
min_num: 0 # minimum number of strings in each value (default 0)
|
||||
max_num: 10 # max number of strings (default to cardinality of source)
|
||||
- name: "a_string_set_from_file"
|
||||
type: "string-set" # (default StringArrayField)
|
||||
source_file: "testdata/file_with_line_delimited_values" # this will be used if not empty
|
||||
generator_type: "random-string" # used to generate random strings rather than pulling from known
|
||||
min_num: 0 # minimum number of strings in each value (default 0)
|
||||
max_num: 10 # max number of strings (default to cardinality of source)
|
||||
- name: "an_id_set"
|
||||
type: "uint-set" # (default IDArrayField)
|
||||
min: 0
|
||||
max: 1000
|
||||
distribution: "uniform"
|
||||
min_num: 1
|
||||
max_num: 3
|
||||
- name: "a_timestamp_field"
|
||||
type: "timestamp" # (default TimestampField)
|
||||
min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
|
||||
max_date: 2010-01-02T15:04:05.001Z # RFC3339Nano
|
||||
distribution: "increasing" # only "increasing" is supported right now
|
||||
min_step_duration: "1s"
|
||||
max_step_duration: "1h" # generated values will add randomly between 1s and 1h to previous value starting at min_date.
|
||||
repeat: false # stop at > max_date unless repeat=true... then go back to min.
|
||||
- name: "a_decimal_field"
|
||||
type: "float" # use idk_params to choose a scale
|
||||
min_float: 0.2
|
||||
max_float: 1.5
|
||||
distribution: "uniform" # only supported value
|
||||
- name: "gender"
|
||||
type: string
|
||||
source_file: "testdata/gender.txt"
|
||||
distribution: "fixed"
|
||||
- name: "shifting_key"
|
||||
type: string
|
||||
# The shifting distribution is meant to simulate a set of strings
|
||||
# that evolves over time. We always generate a number in a
|
||||
# particular range defined by the cardinality, but we always add
|
||||
# an amount to that number which starts at 0 and increases over
|
||||
# time. In this way, some numbers will fall out and never be
|
||||
# generated again, and new numbers will come in. We map each
|
||||
# number to a string by formatting it and padding with zeros to
|
||||
# get the desired length.
|
||||
distribution: "shifting"
|
||||
step: 10 # step for "shifting" distribution means how often do we shift. Every time we generate a value, we'll add an amount to it, every <step> generations, we'll increase the amount by one.
|
||||
cardinality: 123123123
|
||||
max_len: 32
|
||||
s: 1.01
|
||||
v: 200
|
||||
null_chance: 0
|
||||
|
||||
# idk_params describe how data from "fields" should be ingested by IDK
|
||||
idk_params:
|
||||
primary_key_config:
|
||||
field: "id" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up.
|
||||
# fields is keyed by names of fields from top level "fields". It is
|
||||
# not required that all fields appear here, those that don't will
|
||||
# use the default ingestion.
|
||||
fields:
|
||||
an_id:
|
||||
- type: "ID"
|
||||
mutex: false
|
||||
name: "id"
|
||||
a_decimal_field:
|
||||
- type: "Decimal"
|
||||
scale: 4
|
||||
a_timestamp_field:
|
||||
- type: "RecordTime"
|
||||
layout: "2006-01-02T15:04:05Z"
|
||||
epoch: 1970-01-01T00:00:00.0Z
|
||||
name: "na"
|
||||
- type: "Timestamp"
|
||||
name: "a_timestamp_field"
|
||||
# a_string:
|
||||
# - type: "StringField"
|
||||
# keyed: true
|
||||
# mutex: true
|
||||
a_string_set:
|
||||
- type: "StringArray"
|
||||
time_quantum: "YMD"
|
||||
ttl: "0s"
|
||||
5
idk/datagen/testdata/file_with_line_delimited_values
vendored
Normal file
5
idk/datagen/testdata/file_with_line_delimited_values
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
blah
|
||||
blee
|
||||
bloo
|
||||
bleh
|
||||
blin
|
||||
5
idk/datagen/testdata/gender.txt
vendored
Normal file
5
idk/datagen/testdata/gender.txt
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
Male,0.48
|
||||
Female,0.48
|
||||
Transgender,0.01
|
||||
Other,0.01
|
||||
Unspecified,0.02
|
||||
37
idk/datagen/testdata/schema.json
vendored
Normal file
37
idk/datagen/testdata/schema.json
vendored
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
[
|
||||
{
|
||||
"name": "uuid",
|
||||
"path": [
|
||||
"uuid"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "type",
|
||||
"path": [
|
||||
"type"
|
||||
],
|
||||
"type": "id"
|
||||
},
|
||||
{
|
||||
"name": "segid",
|
||||
"path": [
|
||||
"slice"
|
||||
],
|
||||
"type": "ids"
|
||||
},
|
||||
{
|
||||
"name": "timestamp",
|
||||
"path": [
|
||||
"ts"
|
||||
],
|
||||
"type": "recordTime"
|
||||
},
|
||||
{
|
||||
"name": "lastupdated",
|
||||
"path": [
|
||||
"lastupdated"
|
||||
],
|
||||
"type": "timestamp"
|
||||
}
|
||||
]
|
||||
48
idk/datagen/testdata/tremor_keys.yaml
vendored
Normal file
48
idk/datagen/testdata/tremor_keys.yaml
vendored
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
fields:
|
||||
- name: "uuid"
|
||||
type: string
|
||||
distribution: "shifting"
|
||||
step: 10 # step for "shifting" distribution means how often do we shift. Every time we generate a value, we'll add an amount to it, every <step> generations, we'll increase the amount by one.
|
||||
cardinality: 400000000
|
||||
max_len: 32
|
||||
s: 1.01
|
||||
v: 200
|
||||
null_chance: 0
|
||||
- name: "type"
|
||||
type: "int" # (default IntField)
|
||||
min: 0
|
||||
max: 3
|
||||
distribution: "zipfian"
|
||||
s: 1.1
|
||||
v: 5.1
|
||||
- name: "ts"
|
||||
type: "timestamp"
|
||||
min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
|
||||
max_date: 2022-01-02T15:04:05.001Z # RFC3339Nano
|
||||
distribution: "increasing" # only "increasing" is supported right now
|
||||
min_step_duration: "1ms"
|
||||
max_step_duration: "200ms"
|
||||
- name: "lastupdated"
|
||||
type: "timestamp"
|
||||
min_date: 2006-01-02T15:04:05.001Z # RFC3339Nano
|
||||
max_date: 2022-01-02T15:04:05.001Z # RFC3339Nano
|
||||
distribution: "increasing" # only "increasing" is supported right now
|
||||
min_step_duration: "1ms"
|
||||
max_step_duration: "200ms"
|
||||
- name: "slice"
|
||||
type: "uint-set" # (default IDArrayField)
|
||||
min: 0
|
||||
max: 35000
|
||||
distribution: "zipfian"
|
||||
s: 1.1
|
||||
v: 5.1
|
||||
min_num: 1
|
||||
max_num: 50
|
||||
|
||||
# idk_params describe how data from "fields" should be ingested by IDK
|
||||
idk_params:
|
||||
primary_key_config:
|
||||
field: "uuid" # if this is a single field named "id" then we'll use uint IDs, if it's empty we'll autogen ids, and if it's anything else we'll do string keys... yes this is a bit hacky, needs to be cleaned up.
|
||||
# fields is keyed by names of fields from top level "fields". It is
|
||||
# not required that all fields appear here, those that don't will
|
||||
# use the default ingestion.
|
||||
258
idk/datagen/texas_health.go
Normal file
258
idk/datagen/texas_health.go
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/idk/datagen/gen"
|
||||
)
|
||||
|
||||
// Ensure TexasHealth implements interface.
|
||||
var _ Sourcer = (*TexasHealth)(nil)
|
||||
|
||||
// TexasHealth implements Sourcer
|
||||
type TexasHealth struct{}
|
||||
|
||||
// NewTexasHealth returns a new instance of NewTexasHealth.
|
||||
func NewTexasHealth(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &TexasHealth{}
|
||||
}
|
||||
|
||||
type ciscoNetworkEvent struct {
|
||||
id string
|
||||
possibleTypes []string
|
||||
protocol string
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (t *TexasHealth) Source(cfg SourceConfig) idk.Source {
|
||||
|
||||
startTime := time.Date(2020, time.December, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
src := &TexasHealthSource{
|
||||
cur: cfg.startFrom,
|
||||
recordsGenForBucket: 20,
|
||||
currBucketSize: 0,
|
||||
currTimestamp: startTime,
|
||||
endAt: cfg.endAt,
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.IDField{NameVal: "event_id"}, // 1
|
||||
idk.StringField{NameVal: "type"}, // 2
|
||||
idk.StringField{NameVal: "protocol"}, // 3
|
||||
idk.IntField{NameVal: "probe"}, // 4
|
||||
idk.IntField{NameVal: "severity"}, // 5
|
||||
idk.StringField{NameVal: "level"}, // 6
|
||||
idk.StringField{NameVal: "from_interface"}, // 7
|
||||
idk.StringField{NameVal: "from_address"}, // 8
|
||||
idk.StringField{NameVal: "from_port"}, // 9
|
||||
idk.StringField{NameVal: "to_interface"}, // 10
|
||||
idk.StringField{NameVal: "to_address"}, // 11
|
||||
idk.StringField{NameVal: "to_port"}, // 12
|
||||
idk.IntField{NameVal: "duration_time"}, // 13
|
||||
idk.IntField{NameVal: "bytes"}, // 14
|
||||
idk.StringField{NameVal: "hostname"}, // 15
|
||||
idk.TimestampField{NameVal: "timestamp", Unit: "s"}, // 16
|
||||
idk.StringField{NameVal: "timestamp_str"}, // 17
|
||||
},
|
||||
}
|
||||
|
||||
src.g = gen.New(gen.OptGenSeed(cfg.seed))
|
||||
src.possibleEvents = []ciscoNetworkEvent{
|
||||
{
|
||||
id: "302013",
|
||||
possibleTypes: []string{"Built inbound", "Built outbound"},
|
||||
protocol: "TCP",
|
||||
},
|
||||
{
|
||||
id: "302014",
|
||||
possibleTypes: []string{"Teardown"},
|
||||
protocol: "TCP",
|
||||
},
|
||||
{
|
||||
id: "302015",
|
||||
possibleTypes: []string{"Built inbound", "Built outbound"},
|
||||
protocol: "UDP",
|
||||
}, {
|
||||
id: "302016",
|
||||
possibleTypes: []string{"Teardown"},
|
||||
protocol: "UDP",
|
||||
}, {
|
||||
id: "302020",
|
||||
possibleTypes: []string{"Built inbound", "Built outbound"},
|
||||
protocol: "ICMP",
|
||||
}, {
|
||||
id: "302021",
|
||||
possibleTypes: []string{"Teardown"},
|
||||
protocol: "ICMP",
|
||||
},
|
||||
}
|
||||
src.possibleLogLevels = []string{"Debug", "Informational", "Notification", "Warning", "Error", "Critical"}
|
||||
|
||||
// generate hostnames
|
||||
var hostnames []string
|
||||
prefixes := []string{"dbprdfw", "dbrwanasa", "dbdgstsw", "dbrrassa",
|
||||
"phdasa", "dbrprdfw", "hfwasa", "phamasa", "thdrasa", "thalasa",
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
for i := 1; i <= 10; i++ {
|
||||
hostnames = append(hostnames, fmt.Sprintf("%s%02d", prefix, i))
|
||||
}
|
||||
}
|
||||
src.possibleHostnames = hostnames
|
||||
|
||||
// generate ip addresses
|
||||
var ipAddrs []string
|
||||
for i := 1; i <= 100; i++ {
|
||||
ipAddrs = append(ipAddrs, fmt.Sprintf("10.164.124.%d", i))
|
||||
}
|
||||
src.possibleIPAddrs = ipAddrs
|
||||
|
||||
// ports
|
||||
src.possiblePorts = []string{"443", "80", "22", "23", "40", "8080",
|
||||
"53", "88", "115", "123", "143", "20", "21", "25", "101",
|
||||
}
|
||||
|
||||
// interfaces
|
||||
src.possibleFromInterfaces = []string{"faddr",
|
||||
"Security_Cameras_250", "LAB", "PROD",
|
||||
"INSIDE", "provider", "OUTSIDE", "management",
|
||||
"Inside-DataCenter-DBR", "ATT_SBC_Outside", "Outside-WAN-DBR"}
|
||||
src.possibleToInterfaces = []string{"laddr",
|
||||
"OUTSIDE", "Inside-DataCenter", "DBR", "ATT_SBC_Inside", "Outside-WAN-DBR"}
|
||||
|
||||
// rand.Zipf for generating bytes and duration
|
||||
src.randBytes = rand.NewZipf(src.g.R, 1.1, 10, 199)
|
||||
src.randDuration = rand.NewZipf(src.g.R, 1.1, 10, 9999)
|
||||
|
||||
// rest
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[0] = uint64(0)
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (t *TexasHealth) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (t *TexasHealth) DefaultEndAt() uint64 {
|
||||
return 20000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (t *TexasHealth) Info() string {
|
||||
return "Generates sample data for the Texas Health POC."
|
||||
}
|
||||
|
||||
// Ensure TexasHealthSource implements interface.
|
||||
var _ idk.Source = (*TexasHealthSource)(nil)
|
||||
|
||||
// TexasHealthSource is an instance of a source generated
|
||||
// by the Sourcer implementation TexasHleath.
|
||||
type TexasHealthSource struct {
|
||||
g *gen.Gen
|
||||
|
||||
cur uint64
|
||||
endAt uint64
|
||||
|
||||
schema []idk.Field
|
||||
record record
|
||||
|
||||
recordsGenForBucket int
|
||||
currBucketSize int
|
||||
currTimestamp time.Time
|
||||
|
||||
randBytes *rand.Zipf
|
||||
randDuration *rand.Zipf
|
||||
|
||||
possibleEvents []ciscoNetworkEvent
|
||||
possibleLogLevels []string
|
||||
possibleHostnames []string
|
||||
possibleIPAddrs []string
|
||||
possiblePorts []string
|
||||
possibleFromInterfaces []string
|
||||
possibleToInterfaces []string
|
||||
}
|
||||
|
||||
func (s *TexasHealthSource) genTimestamp() time.Time {
|
||||
return s.currTimestamp
|
||||
}
|
||||
|
||||
// advanceTick is a helper method to do some housekeeping
|
||||
// after a record is generated. For one, it increments cur
|
||||
// which keeps track of the number of records generated.
|
||||
// Additionally, it advances the 'time-slot buckets' for
|
||||
// lack of better naming. Each record generated belongs to
|
||||
// a given time bucket and within that time bucket, all the
|
||||
// records bear the same timestamp value (at second resolution).
|
||||
// The bucket sizes are randomized and generally range between 1
|
||||
// to 200. Currently, the bucket size value is generated randomly
|
||||
// from a uniform distribution but a more realistic datagen should
|
||||
// weight this value.
|
||||
func (s *TexasHealthSource) advanceTick() {
|
||||
s.cur++
|
||||
s.recordsGenForBucket++
|
||||
|
||||
if s.recordsGenForBucket >= s.currBucketSize {
|
||||
// reset bucket size to a random value between 1 and 200
|
||||
s.currBucketSize = s.g.R.Intn(200) + 1
|
||||
s.recordsGenForBucket = 0
|
||||
oneSecond := time.Duration(1e9)
|
||||
s.currTimestamp = s.currTimestamp.Add(oneSecond)
|
||||
}
|
||||
}
|
||||
|
||||
// Record implements idk.Source.
|
||||
func (s *TexasHealthSource) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
selectEventIndex := s.g.R.Intn(len(s.possibleEvents))
|
||||
event := s.possibleEvents[selectEventIndex]
|
||||
logLevelIndex := s.g.R.Intn(len(s.possibleLogLevels))
|
||||
timestamp := s.genTimestamp()
|
||||
|
||||
s.record[0] = s.cur // id
|
||||
s.record[1] = event.id // event_id
|
||||
s.record[2] = s.g.StringFromList(event.possibleTypes) // type
|
||||
s.record[3] = event.protocol // protocol
|
||||
s.record[4] = 3457177187 + s.cur // probe
|
||||
s.record[5] = logLevelIndex + 1 // severity
|
||||
s.record[6] = s.possibleLogLevels[logLevelIndex] // level
|
||||
s.record[7] = s.g.StringFromList(s.possibleFromInterfaces) // from_interface
|
||||
s.record[8] = s.g.StringFromListWeighted(s.possibleIPAddrs) // from_address
|
||||
s.record[9] = s.g.StringFromListWeighted(s.possiblePorts) // from_port
|
||||
s.record[10] = s.g.StringFromList(s.possibleToInterfaces) // to_interface
|
||||
s.record[11] = s.g.StringFromListWeighted(s.possibleIPAddrs) // to_address
|
||||
s.record[12] = s.g.StringFromListWeighted(s.possiblePorts) // to_port
|
||||
s.record[13] = s.randDuration.Uint64() + 1 // duration_time
|
||||
s.record[14] = s.randBytes.Uint64() + 1 // bytes
|
||||
s.record[15] = s.g.StringFromList(s.possibleHostnames) // hostname
|
||||
s.record[16] = timestamp.Unix() // timestamp
|
||||
s.record[17] = timestamp.String() // timestamp_str
|
||||
|
||||
s.advanceTick()
|
||||
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
// Schema implements idk.Source.
|
||||
func (s *TexasHealthSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
func (s *TexasHealthSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
188
idk/datagen/timeseries.go
Normal file
188
idk/datagen/timeseries.go
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Timeseries implements interface.
|
||||
var _ Sourcer = (*Timeseries)(nil)
|
||||
|
||||
// Timeseries implements Sourcer.
|
||||
type Timeseries struct {
|
||||
schema []idk.Field
|
||||
}
|
||||
|
||||
// NewTimeseries returns a new instance of Timeseries.
|
||||
func NewTimeseries(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Timeseries{
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "data_type"}, // 1
|
||||
idk.IntField{NameVal: "equip_id"}, // 2
|
||||
idk.StringField{NameVal: "group"}, // 3
|
||||
idk.IntField{NameVal: "int_value"}, // 4
|
||||
idk.StringField{NameVal: "string_value"}, // 5
|
||||
idk.IntField{NameVal: "timestamp"}, // 6
|
||||
idk.StringField{NameVal: "type"}, // 7
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (t *Timeseries) Source(cfg SourceConfig) idk.Source {
|
||||
src := &TimeseriesSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
|
||||
schema: t.schema,
|
||||
}
|
||||
|
||||
src.typeZipf = rand.NewZipf(src.rand, 1.03, 4, uint64(len(tsTypes))-1)
|
||||
src.tsZipf = rand.NewZipf(src.rand, 3, 1.3, uint64(len(tsStrings))-1)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[6] = int(1420070400 + cfg.startFrom)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (t *Timeseries) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (t *Timeseries) DefaultEndAt() uint64 {
|
||||
return 100000000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (t *Timeseries) Info() string {
|
||||
return "Generates timeseries device power state data (device, alert, status) - references data in sites, manufacturer, and equipment."
|
||||
}
|
||||
|
||||
// Ensure TimeseriesSource implements interface.
|
||||
var _ idk.Source = (*TimeseriesSource)(nil)
|
||||
|
||||
type TimeseriesSource struct {
|
||||
Log logger.Logger
|
||||
|
||||
rand *rand.Rand
|
||||
typeZipf *rand.Zipf
|
||||
tsZipf *rand.Zipf
|
||||
|
||||
schema []idk.Field
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
record record
|
||||
}
|
||||
|
||||
func NewTimeseriesSource(start, end uint64) *TimeseriesSource {
|
||||
src := &TimeseriesSource{
|
||||
cur: start,
|
||||
endAt: end,
|
||||
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "data_type"}, // 1
|
||||
idk.IntField{NameVal: "equip_id"}, // 2
|
||||
idk.StringField{NameVal: "group"}, // 3
|
||||
idk.IntField{NameVal: "int_value"}, // 4
|
||||
idk.StringField{NameVal: "string_value"}, // 5
|
||||
idk.IntField{NameVal: "timestamp"}, // 6
|
||||
idk.StringField{NameVal: "type"}, // 7
|
||||
},
|
||||
}
|
||||
|
||||
src.typeZipf = rand.NewZipf(src.rand, 1.03, 4, uint64(len(tsTypes))-1)
|
||||
src.tsZipf = rand.NewZipf(src.rand, 3, 1.3, uint64(len(tsStrings))-1)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[6] = int(1420070400 + start)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
func (s *TimeseriesSource) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
s.record[0] = s.cur
|
||||
|
||||
// type
|
||||
s.record[7] = tsTypes[s.typeZipf.Uint64()]
|
||||
groups, ok := groupsPerType[s.record[7].(string)]
|
||||
if !ok {
|
||||
s.Log.Printf("no %v in groups", s.record[7])
|
||||
}
|
||||
|
||||
// group
|
||||
if len(groups) != 0 {
|
||||
s.record[3] = groups[s.rand.Intn(len(groups))]
|
||||
} else {
|
||||
s.record[3] = nil
|
||||
}
|
||||
|
||||
// equip_id
|
||||
s.record[2] = int64(s.rand.Intn(5000000))
|
||||
|
||||
// data type and string or int value
|
||||
if s.rand.Intn(1000) > 995 {
|
||||
s.record[1] = "string"
|
||||
s.record[4] = nil
|
||||
s.record[5] = tsStrings[s.tsZipf.Uint64()]
|
||||
} else {
|
||||
s.record[1] = "int"
|
||||
s.record[4] = s.rand.Intn(100000)
|
||||
s.record[5] = nil
|
||||
}
|
||||
|
||||
// timestamp
|
||||
s.record[6] = s.record[6].(int) + s.rand.Intn(3)
|
||||
|
||||
s.cur++
|
||||
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *TimeseriesSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *TimeseriesSource) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*TimeseriesSource)(nil)
|
||||
|
||||
var tsStrings = []string{"on", "off", "alarm", "safe"}
|
||||
|
||||
var groupsPerType = map[string][]string{
|
||||
"Voltage": {"calculated", "single", "moving"},
|
||||
"VoltageRMS": {"calculated", "single", "moving"},
|
||||
"Amperage": {"calculated", "single", "effective"},
|
||||
"Resistance": {"fixed", "variable"},
|
||||
"Frequency": {"sampled", "digital", "analog"},
|
||||
"Message": {"event", "alert", "alarm"},
|
||||
}
|
||||
|
||||
func (s *TimeseriesSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
5005
idk/datagen/titles.data.go
Normal file
5005
idk/datagen/titles.data.go
Normal file
File diff suppressed because it is too large
Load diff
207
idk/datagen/transactions.go
Normal file
207
idk/datagen/transactions.go
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Transaction implements interface.
|
||||
var _ Sourcer = (*Transaction)(nil)
|
||||
|
||||
// Transaction implements Sourcer.
|
||||
type Transaction struct {
|
||||
schema []idk.Field
|
||||
}
|
||||
|
||||
// NewTransaction returns a new instance of Transaction.
|
||||
func NewTransaction(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Transaction{
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "transaction_type"}, // 1
|
||||
idk.StringField{NameVal: "payment_type"}, // 2
|
||||
idk.IntField{NameVal: "site_id"}, // 3
|
||||
idk.IntField{NameVal: "equip_id"}, // 4
|
||||
idk.StringField{NameVal: "customer"}, // 5
|
||||
idk.DecimalField{NameVal: "transaction_value", Scale: 2}, // 6
|
||||
idk.IntField{NameVal: "timestamp"}, // 7
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (t *Transaction) Source(cfg SourceConfig) idk.Source {
|
||||
src := &TransactionSource{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
|
||||
schema: t.schema,
|
||||
}
|
||||
|
||||
src.typeZipf = rand.NewZipf(src.rand, 1.03, 4, uint64(len(tsTypes))-1)
|
||||
src.instoreZipf = rand.NewZipf(src.rand, 1.3, 1.3, uint64(len(inStorePayments))-1)
|
||||
src.ecomZipf = rand.NewZipf(src.rand, 1.3, 1.3, uint64(len(ecomPayments))-1)
|
||||
src.valueZipf = rand.NewZipf(src.rand, 1.2, 4, 50000)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[7] = int(1420070400 + cfg.startFrom)
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (t *Transaction) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (t *Transaction) DefaultEndAt() uint64 {
|
||||
return 100000000
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (t *Transaction) Info() string {
|
||||
return "Generates timeseries transaction data (transaction_amount, POS, site) - references data in sites, manufacturer, and equipment."
|
||||
}
|
||||
|
||||
// Ensure TransactionSource implements interface.
|
||||
var _ idk.Source = (*TransactionSource)(nil)
|
||||
|
||||
type TransactionSource struct {
|
||||
Log logger.Logger
|
||||
|
||||
rand *rand.Rand
|
||||
typeZipf *rand.Zipf
|
||||
instoreZipf *rand.Zipf
|
||||
ecomZipf *rand.Zipf
|
||||
valueZipf *rand.Zipf
|
||||
|
||||
schema []idk.Field
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
record record
|
||||
}
|
||||
|
||||
var transactionTypes = []string{"Ecommerce", "In-store"}
|
||||
|
||||
func (s *TransactionSource) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = s.cur
|
||||
s.cur++
|
||||
|
||||
// transaction type
|
||||
transType := transactionTypes[s.rand.Intn(2)]
|
||||
s.record[1] = transType
|
||||
|
||||
// payment type
|
||||
paymentType := s.paymentType(transType)
|
||||
s.record[2] = paymentType
|
||||
|
||||
// site id
|
||||
s.record[3] = s.siteID(transType)
|
||||
|
||||
// equipment id
|
||||
s.record[4] = s.equipID(transType)
|
||||
|
||||
// customer
|
||||
s.record[5] = s.customer()
|
||||
|
||||
// transaction_value
|
||||
s.record[6] = s.value(paymentType)
|
||||
|
||||
// timestamp
|
||||
s.record[7] = s.record[7].(int) + s.rand.Intn(3)
|
||||
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *TransactionSource) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *TransactionSource) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*TransactionSource)(nil)
|
||||
|
||||
func (s *TransactionSource) paymentType(transType string) string {
|
||||
switch {
|
||||
case transType == "Ecommerce":
|
||||
return ecomPayments[s.ecomZipf.Uint64()]
|
||||
case transType == "In-store":
|
||||
return inStorePayments[s.instoreZipf.Uint64()]
|
||||
default:
|
||||
panic("unknown transaction type " + transType)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TransactionSource) siteID(transType string) int64 {
|
||||
switch {
|
||||
case transType == "Ecommerce":
|
||||
num := s.rand.Int63n(500000)
|
||||
if num < 400000 {
|
||||
return num + 88000 // retail locations
|
||||
} else {
|
||||
return num + 4000000 // subset of data centers
|
||||
}
|
||||
case transType == "In-store":
|
||||
return s.rand.Int63n(400000) + 88000
|
||||
default:
|
||||
panic("unknown transaction type (siteid)" + transType)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TransactionSource) equipID(transType string) int64 {
|
||||
switch {
|
||||
case transType == "Ecommerce":
|
||||
return s.rand.Int63n(3000000) + 20000000 // subset of servers
|
||||
case transType == "In-store":
|
||||
return s.rand.Int63n(1000000) + 35000000 // POS devices
|
||||
default:
|
||||
panic("unknown transaction type (equipid)" + transType)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TransactionSource) customer() string {
|
||||
return customers[s.rand.Intn(len(customers))]
|
||||
}
|
||||
|
||||
func (s *TransactionSource) value(paymentType string) float64 {
|
||||
val := float64(s.valueZipf.Uint64() + 15) // 15-50k
|
||||
|
||||
// 25% go lower ... to get some below $15
|
||||
if s.rand.Intn(4) == 0 {
|
||||
val -= float64(s.rand.Intn(14))
|
||||
}
|
||||
|
||||
// half the time add cents
|
||||
if s.rand.Intn(2) == 1 {
|
||||
val = val + s.rand.Float64()
|
||||
}
|
||||
|
||||
if paymentType == "Return" {
|
||||
val = val * -1
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
func (s *TransactionSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
203
idk/datagen/transactions_scenario_1.go
Normal file
203
idk/datagen/transactions_scenario_1.go
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
package datagen
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/rand"
|
||||
|
||||
"github.com/molecula/featurebase/v3/idk"
|
||||
"github.com/molecula/featurebase/v3/logger"
|
||||
)
|
||||
|
||||
// Ensure Transaction1 implements interface.
|
||||
var _ Sourcer = (*Transaction1)(nil)
|
||||
|
||||
// Transaction1 implements Sourcer.
|
||||
type Transaction1 struct {
|
||||
schema []idk.Field
|
||||
}
|
||||
|
||||
// NewTransaction1 returns a new instance of Transaction1.
|
||||
func NewTransaction1(cfg SourceGeneratorConfig) Sourcer {
|
||||
return &Transaction1{
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "transaction_type"}, // 1
|
||||
idk.StringField{NameVal: "payment_type"}, // 2
|
||||
idk.IntField{NameVal: "site_id"}, // 3
|
||||
idk.IntField{NameVal: "equip_id"}, // 4
|
||||
idk.StringField{NameVal: "customer"}, // 5
|
||||
idk.DecimalField{NameVal: "transaction_value", Scale: 2}, // 6
|
||||
idk.IntField{NameVal: "timestamp"}, // 7
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Source returns an idk.Source which will generate
|
||||
// records for a partition of the entire record space,
|
||||
// determined by the concurrency value. It implements
|
||||
// the Sourcer interface.
|
||||
func (t *Transaction1) Source(cfg SourceConfig) idk.Source {
|
||||
src := &Transaction1Source{
|
||||
cur: cfg.startFrom,
|
||||
endAt: cfg.endAt,
|
||||
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
|
||||
schema: t.schema,
|
||||
}
|
||||
|
||||
src.instoreZipf = rand.NewZipf(src.rand, 1.3, 1.3, uint64(len(inStorePayments))-1)
|
||||
src.ecomZipf = rand.NewZipf(src.rand, 1.3, 1.3, uint64(len(ecomPayments))-1)
|
||||
src.valueZipf = rand.NewZipf(src.rand, 1.8, 4, 400)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[7] = int(1509100000 + uint64(59.5*float64(cfg.startFrom)))
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
// PrimaryKeyFields returns the fields from the schema
|
||||
// which should be used as the index's primary key.
|
||||
func (t *Transaction1) PrimaryKeyFields() []string {
|
||||
return []string{"id"}
|
||||
}
|
||||
|
||||
// DefaultEndAt sets the endAt record value for the
|
||||
// case where one is not provided. It implements the
|
||||
// Sourcer interface.
|
||||
func (t *Transaction1) DefaultEndAt() uint64 {
|
||||
return 23682459
|
||||
}
|
||||
|
||||
// Info describes what this implementation of Sourcer
|
||||
// generates. It implements the Sourcer interface.
|
||||
func (t *Transaction1) Info() string {
|
||||
return "TODO"
|
||||
}
|
||||
|
||||
// Ensure Transaction1Source implements interface.
|
||||
var _ idk.Source = (*Transaction1Source)(nil)
|
||||
|
||||
type Transaction1Source struct {
|
||||
Log logger.Logger
|
||||
|
||||
cur, endAt uint64
|
||||
|
||||
rand *rand.Rand
|
||||
instoreZipf *rand.Zipf
|
||||
ecomZipf *rand.Zipf
|
||||
valueZipf *rand.Zipf
|
||||
|
||||
schema []idk.Field
|
||||
|
||||
record record
|
||||
}
|
||||
|
||||
func NewTransaction1Source(start, end uint64) *Transaction1Source {
|
||||
src := &Transaction1Source{
|
||||
cur: start,
|
||||
endAt: end,
|
||||
|
||||
rand: rand.New(rand.NewSource(22)),
|
||||
|
||||
schema: []idk.Field{
|
||||
idk.IDField{NameVal: "id"}, // 0
|
||||
idk.StringField{NameVal: "transaction_type"}, // 1
|
||||
idk.StringField{NameVal: "payment_type"}, // 2
|
||||
idk.IntField{NameVal: "site_id"}, // 3
|
||||
idk.IntField{NameVal: "equip_id"}, // 4
|
||||
idk.StringField{NameVal: "customer"}, // 5
|
||||
idk.DecimalField{NameVal: "transaction_value", Scale: 2}, // 6
|
||||
idk.IntField{NameVal: "timestamp"}, // 7
|
||||
},
|
||||
}
|
||||
|
||||
src.instoreZipf = rand.NewZipf(src.rand, 1.3, 1.3, uint64(len(inStorePayments))-1)
|
||||
src.ecomZipf = rand.NewZipf(src.rand, 1.3, 1.3, uint64(len(ecomPayments))-1)
|
||||
src.valueZipf = rand.NewZipf(src.rand, 1.8, 4, 400)
|
||||
|
||||
src.record = make([]interface{}, len(src.schema))
|
||||
src.record[7] = int(1509100000 + uint64(59.5*float64(start)))
|
||||
|
||||
return src
|
||||
}
|
||||
|
||||
func (s *Transaction1Source) Record() (idk.Record, error) {
|
||||
if s.cur >= s.endAt {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[0] = 100000000 + s.cur
|
||||
|
||||
// timestamp
|
||||
timestamp := s.record[7].(int) + s.rand.Intn(120)
|
||||
if timestamp > 1509106300 {
|
||||
return nil, io.EOF
|
||||
}
|
||||
s.record[7] = timestamp
|
||||
|
||||
// transaction type
|
||||
transType := "In-store"
|
||||
s.record[1] = transType
|
||||
|
||||
// payment type
|
||||
paymentType := s.paymentType(transType)
|
||||
s.record[2] = paymentType
|
||||
|
||||
// site id
|
||||
s.record[3] = 88002 // Retail site which has breaker 712278
|
||||
|
||||
// equipment id
|
||||
s.record[4] = []uint64{35100914, 35138991, 35144935, 35329399, 35894415}[s.cur%5] // POSs
|
||||
|
||||
// customer
|
||||
s.record[5] = "CVS"
|
||||
|
||||
// transaction_value
|
||||
s.record[6] = s.value(paymentType)
|
||||
|
||||
s.cur++
|
||||
|
||||
return s.record, nil
|
||||
}
|
||||
|
||||
func (s *Transaction1Source) Schema() []idk.Field {
|
||||
return s.schema
|
||||
}
|
||||
|
||||
func (s *Transaction1Source) Seed(seed int64) {
|
||||
s.rand.Seed(seed)
|
||||
}
|
||||
|
||||
var _ Seedable = (*Transaction1Source)(nil)
|
||||
|
||||
func (s *Transaction1Source) paymentType(transType string) string {
|
||||
switch {
|
||||
case transType == "Ecommerce":
|
||||
return ecomPayments[s.ecomZipf.Uint64()]
|
||||
case transType == "In-store":
|
||||
return inStorePayments[s.instoreZipf.Uint64()]
|
||||
default:
|
||||
panic("unknown transaction type " + transType)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Transaction1Source) value(paymentType string) float64 {
|
||||
val := float64(s.valueZipf.Uint64() + 15) // 15-50k
|
||||
|
||||
// 25% go lower ... to get some below $15
|
||||
if s.rand.Intn(4) == 0 {
|
||||
val -= float64(s.rand.Intn(14))
|
||||
}
|
||||
|
||||
// add cents
|
||||
val = val + s.rand.Float64()
|
||||
|
||||
if paymentType == "Return" {
|
||||
val = val * -1
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
func (s *Transaction1Source) Close() error {
|
||||
return nil
|
||||
}
|
||||
28342
idk/datagen/uscities.data.go
Normal file
28342
idk/datagen/uscities.data.go
Normal file
File diff suppressed because it is too large
Load diff
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue