scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 08:35:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
featurebase/idk/Makefile
jonathan.cheng 43f63b93e3
Cloud 940, skip bad row, enable ingest subsequent rows (#2201) 
* first try to skip decodeMessage() error

* force idk to skip a row if there are errors in recordizing

* add comment for removing returning errors from decodemessage()

* added ingest flag SkipBadRows and implementation for skipping Bad Rows (errors that come from recordizer)

* added some comments

* removed comment

* made changes as per discussion with jaffee and walter. i hope this works...

* adding unit tests to check functionality implemented for CLOUD-940

* addressing review comments

* changed a variable name in test file

* removed a variable from ingest test file

* testing sonarcloud failure

* drop spurious second sonar-scanner call

We call sonar-scanner on the IDK data, and then we change
into the IDK directory and try to run it again on the same files,
which don't exist.

* abandon idk change detection for now

the "changes" rule appears not to be good at detecting changes
in some cases. specifically, it appears that you have to be in
an "only:" clause, not a "rules" clause, to trigger the
merge-specific behavior which checks the entire merge branch
instead of the top commit, but that means that if your last
commit doesn't touch IDK, we don't run IDK tests, and I haven't
been able to fix this yet.

So for now, revert the IDK-specific change detection behavior,
which slows CI down but gets us test coverage.

* fix path references

we had three tests all creating idk_coverage.out, then we tried
to grab all files named coverage.out from the testdata directory.

* refactoring tests to avoid duplication

* reverting changes made for local testing

Co-authored-by: CHIN JUNG CHENG <chengcj@CHINs-MacBook-Pro.local>
Co-authored-by: Pranitha-malae <56414132+Pranitha-malae@users.noreply.github.com>
Co-authored-by: Pranitha-malae <pranitha453@gmail.com>
Co-authored-by: Seebs <seebs@molecula.com>
2022-10-06 16:20:35 -05:00

330 lines
15 KiB
Makefile
Raw Blame History

GO ?= go
GOOS ?= $(shell $(GO) env GOOS)
GOARCH ?= $(shell $(GO) env GOARCH)
GOPATH ?= $(shell $(GO) env GOPATH)
GO_VERSION=1.19
GO_BUILD_FLAGS=
ODBC_ENABLED=
BINOUT ?= bin
VERSION := $(shell git describe --tags 2> /dev/null || git rev-parse --verify --short=7 HEAD)
VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
BRANCH_NAME ?= ""
# We allow setting a custom docker-compose "project". Multiple of the
# same docker-compose environment can exist simultaneously as long as
# they use different projects (the project name is prepended to
# container names and such). This is useful in a CI environment where
# we might be running multiple instances of the tests concurrently.
PROJECT ?= idk
DOCKER_COMPOSE = docker-compose -p $(PROJECT)
UNAME_P := $(shell uname -p)
ifneq ($(filter arm%,$(UNAME_P)),)
IS_ARM := 1
else
IS_ARM := 0
endif
ifneq ($(filter Darwin%,$(shell uname)),)
IS_MAC := 1
else
IS_MAC := 0
endif
LDFLAGS="-X 'github.com/molecula/featurebase/v3/idk.Version=$(VERSION)' -X 'github.com/molecula/featurebase/v3/idk.BuildTime=$(BUILD_TIME)'"
LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X 'github.com/molecula/featurebase/v3/idk.Version=$(VERSION)' -X 'github.com/molecula/featurebase/v3/idk.BuildTime=$(BUILD_TIME)' "
export GOPRIVATE=github.com/molecula
export CGO_ENABLED=1
# Cloud Deployment environment (dev, prod, staging, etc.), specified with the value of MCLOUD_ENV
# E.g. `MCLOUD_ENV=test make docker-push-ecr`
MCLOUD_ENV ?= dev
MCLOUD_ENV_FILE=.cloud-env.$(MCLOUD_ENV)
-include $(MCLOUD_ENV_FILE)
PROFILE ?=
AWS_REGION ?=
ECR_URL ?=
ECR_REPO ?= idk
BINS := $(patsubst cmd/%,$(BINOUT)/%,$(wildcard cmd/*))
.PHONY: install build release-build release test $(BINS)
install:
ifeq ($(filter $(IS_ARM),$(IS_MAC)), 1)
export HOMEBREW_NO_AUTO_UPDATE=1 && [ "${PKG_CONFIG_PATH}" != "$$(brew --prefix openssl)/lib/pkgconfig" ] && (brew list pkg-config || brew install pkg-config) && (brew list librdkafka || brew install librdkafka) && (brew list openssl || brew install openssl) && export PKG_CONFIG_PATH="$$(brew --prefix openssl)/lib/pkgconfig"; \
$(GO) install -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -tags dynamic ./cmd/...
else
$(GO) install -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) ./cmd/...
endif
$(BINS):
$(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o $@ ./cmd/$(notdir $@)
$(BINOUT)/molecula-consumer-sql-odbc:
CGO_ENABLED=1 $(GO) build -tags=odbc -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o $@ ./cmd/molecula-consumer-sql
build_non_cgo:
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/ingester ./cmd/ingester
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-csv ./cmd/molecula-consumer-csv
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-static ./cmd/molecula-consumer-kafka-static
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-sql ./cmd/molecula-consumer-sql
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-github ./cmd/molecula-consumer-github
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kinesis ./cmd/molecula-consumer-kinesis
build_cgo:
ifeq ($(GOARCH), arm64)
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka ./cmd/molecula-consumer-kafka
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-delete ./cmd/molecula-consumer-kafka-delete
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/kafkagen ./cmd/kafkagen
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/kafkaput ./cmd/kafkaput
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/datagen ./cmd/datagen
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/bankgen ./cmd/bankgen
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-sasl ./cmd/molecula-consumer-kafka-sasl
endif
ifeq ($(GOARCH), amd64)
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka ./cmd/molecula-consumer-kafka
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-delete ./cmd/molecula-consumer-kafka-delete
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/kafkagen ./cmd/kafkagen
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/kafkaput ./cmd/kafkaput
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/datagen ./cmd/datagen
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/bankgen ./cmd/bankgen
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-sasl ./cmd/molecula-consumer-kafka-sasl
endif
CGO_ENABLED=1 $(GO) build -tags=odbc -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-sql-odbc ./cmd/molecula-consumer-sql
BUILD_CGO ?= 0
build:
@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
make build_non_cgo
ifeq ($(BUILD_CGO), 1)
make build_cgo
endif
release-build:
mkdir -p build/molecula-consumer-$(VERSION_ID)/
$(MAKE) build BINOUT=build/molecula-consumer-$(VERSION_ID)/
tar -cvz -C build -f build/molecula-consumer-$(VERSION_ID).tar.gz molecula-consumer-$(VERSION_ID)/
@echo Created release build: build/molecula-consumer-$(VERSION_ID).tar.gz
release: check-clean
make release-build GOOS=linux GOARCH=amd64
make release-build GOOS=linux GOARCH=arm64
make release-build GOOS=darwin GOARCH=amd64
make release-build GOOS=darwin GOARCH=arm64
test:
$(GO) test ./...
test-local: testenv
IDK_TEST_CERT_PATH=$(shell pwd)/testenv/certs \
IDK_TEST_KAFKA_HOST=localhost:9092 \
IDK_TEST_PILOSA_GRPC_HOST=localhost:20101 \
IDK_TEST_PILOSA_HOST=localhost:10101 \
IDK_TEST_PILOSA_TLS_HOST=https://localhost:10111 \
IDK_TEST_REGISTRY_HOST=localhost:8081 \
$(GO) test $(TESTFLAGS) ./...
vendor: ../go.mod
$(GO) mod vendor
build-%:
$(DOCKER_COMPOSE) build $*
pull-%:
$(DOCKER_COMPOSE) pull $*
save-%-logs:
$(DOCKER_COMPOSE) logs $* > ./testdata/$(PROJECT)_$*_logs.txt
start-all: .pulled testenv build-wait
echo "branch name" ${BRANCH_NAME}
$(DOCKER_COMPOSE) up -d zookeeper
$(DOCKER_COMPOSE) run -T wait zookeeper 'echo "ruok" | nc -w 2 zookeeper 2181 | grep imok'
$(DOCKER_COMPOSE) up -d kafka
$(DOCKER_COMPOSE) up -d schema-registry
$(DOCKER_COMPOSE) up -d postgres
$(DOCKER_COMPOSE) run -T wait postgres pg_isready -h postgres -p 5432 -U postgres
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa-tls
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa-auth
$(DOCKER_COMPOSE) run -T wait pilosa curl --silent --fail http://pilosa:10101/status
$(DOCKER_COMPOSE) run -T wait pilosa-tls curl --silent --cacert /certs/ca.crt --key /certs/theclient.key --cert /certs/theclient.crt --fail https://pilosa-tls:10111/status
$(DOCKER_COMPOSE) run -T wait pilosa-auth curl --silent --fail http://pilosa-auth:10105/version
$(DOCKER_COMPOSE) run -T wait kafka nc -z kafka 9092
$(DOCKER_COMPOSE) run -T wait schema-registry curl --silent --fail http://schema-registry:8081/config
start-postgres: build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d postgres
$(DOCKER_COMPOSE) run -T wait postgres pg_isready -h postgres -p 5432 -U postgres
start-pilosa: build-pilosa start-postgres build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d pilosa
$(DOCKER_COMPOSE) run -T wait pilosa curl --silent --fail http://pilosa:10101/status
start-pilosa-tls: build-pilosa-tls build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d pilosa-tls
$(DOCKER_COMPOSE) run -T wait pilosa-tls curl --silent --cacert /certs/ca.crt --key /certs/theclient.key --cert /certs/theclient.crt --fail https://pilosa-tls:10111/status
start-pilosa-auth: build-pilosa-auth build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d pilosa-auth
$(DOCKER_COMPOSE) run -T wait pilosa-auth curl --silent --fail http://pilosa-auth:10105/version
start-zookeeper: pull-zookeeper testenv
$(DOCKER_COMPOSE) up -d zookeeper
$(DOCKER_COMPOSE) run -T wait zookeeper 'echo "ruok" | nc -w 2 zookeeper 2181 | grep imok'
start-kafka: pull-kafka start-zookeeper build-wait testenv
$(DOCKER_COMPOSE) up -d kafka
$(DOCKER_COMPOSE) run -T wait kafka nc -z kafka 9092
start-schema-registry: start-zookeeper start-kafka pull-schema-registry build-wait testenv
$(DOCKER_COMPOSE) up -d schema-registry
$(DOCKER_COMPOSE) run -T wait schema-registry curl --silent --fail http://schema-registry:8081/config
startup-old: start-postgres start-pilosa start-pilosa-tls start-pilosa-auth start-zookeeper start-kafka start-schema-registry
startup: start-all
shutdown:
$(DOCKER_COMPOSE) down -v --remove-orphans
rm -f .pulled
test-all: testenv
$(MAKE) startup
$(MAKE) test-run
$(DOCKER_COMPOSE) logs kafka | grep "Kafka version"
$(MAKE) shutdown
test-all-race: testenv
$(MAKE) startup
$(MAKE) test-run-race
$(MAKE) shutdown
test-all-kafka-sasl: testenv
$(MAKE) startup
$(MAKE) test-run-kafka-sasl
$(MAKE) shutdown
TCMD ?= ./...
# do "make startup", then e.g. "make test-run-local TCMD='-run=MyFavTest ./kafka'"
test-run-local:
pwd
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test go test -mod=vendor -tags=odbc,dynamic $(TCMD)
TPKG ?= ./...
test-run: testenv vendor
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -json -coverprofile=/testdata/$(PROJECT)_base_coverage.out | tee /testdata/$(PROJECT)_report.out"
$(DOCKER_COMPOSE) run -T idk-test /go/src/github.com/molecula/featurebase/idk/reingest_test.sh
test-run-race: testenv vendor
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -race -covermode=atomic -tags=dynamic $(TPKG) -coverpkg=$(TPKG) -timeout=30m -json -coverprofile=/testdata/$(PROJECT)_race_coverage.out | tee /testdata/$(PROJECT)_report.out"
test-run-kafka-sasl: testenv vendor
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v --tags=kafka_sasl -mod=vendor -race -timeout=30m $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -json -coverprofile=/testdata/$(PROJECT)_sasl_coverage.out | tee /testdata/$(PROJECT)_report.out"
.pulled:
$(DOCKER_COMPOSE) pull
touch .pulled
testenv: testenv/certs
testenv/certs:
certstrap --depot-path testenv/certs init --common-name ca --passphrase "" --expires "100 years"
certstrap --depot-path=testenv/certs request-cert --domain localhost --common-name localhost --passphrase ""
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" localhost
certstrap --depot-path=testenv/certs request-cert --domain pilosa-tls --common-name pilosa-tls --passphrase ""
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" pilosa-tls
certstrap --depot-path=testenv/certs request-cert --domain theclient --common-name theclient --passphrase ""
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" theclient
# Create release using Docker
docker-release:
@if [ $(IS_ARM) = 1 ]; then \
echo Building linux-amd64 image on $(UNAME_P) is not supported; \
else \
$(MAKE) docker-build GOOS=linux GOARCH=amd64; \
fi
$(MAKE) docker-build GOOS=darwin GOARCH=amd64
$(MAKE) docker-build GOOS=darwin GOARCH=arm64
$(MAKE) docker-build GOOS=linux GOARCH=arm64
# This allows multiple concurrent builds to happen in CI without
# creating container name conflicts and such. (different BUILD_NAMEs
# are passed in from gitlab-ci.yml)
BUILD_NAME ?= idk-build
# Build a release in Docker
docker-build: vendor
DOCKER_BUILDKIT=0 docker build \
-f ../idk/Dockerfile \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
--build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
--target builder \
--tag idk:$(BUILD_NAME) ../.
mkdir -p build/idk-$(GOOS)-$(GOARCH)
docker create --name $(BUILD_NAME) idk:$(BUILD_NAME)
docker cp $(BUILD_NAME):/featurebase/idk/bin/. ./build/idk-$(GOOS)-$(GOARCH)
docker rm $(BUILD_NAME)
# Create Docker image from Dockerfile
docker-image: vendor
docker build \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
--tag registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) .
@echo Created docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
docker: docker-image
docker-tag-push:
docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
@echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
clean:
$(MAKE) shutdown
rm -rf testenv
# Error out if there are untracked changes in Git
check-clean:
ifndef SKIP_CHECK_CLEAN
$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
endif
docker-push-ecr: docker-image aws-login
docker tag idk:$(VERSION) $(ECR_URL)/$(ECR_REPO):$(VERSION)
aws ecr get-login-password --region $(AWS_REGION) --profile $(PROFILE)| docker login --username AWS --password-stdin $(ECR_URL)
docker push $(ECR_URL)/$(ECR_REPO):$(VERSION)
aws-login:
aws sso login --profile $(PROFILE)
# Build mock implementations of AWS service API interfaces for unit testing.
#
# These are run-once/rarely targets as they produce code artifacts only used
# during `go test`. An automated CI/CD process does NOT need to invoke every time.
MOCKS = S3 Kinesis SQS
install-mock-generator:
@which $(GOPATH)/bin/mockery || (echo "Installing missing dependency 'mockery' to generate mock implementations for unit testing." && \
$(GO) install 'github.com/vektra/mockery/v2@latest')
update-mocks: $(addprefix update-mocks-, $(MOCKS))
update-mocks-%: install-mock-generator
$(eval AWS_SERVICE := $(shell echo $* | tr '[:upper:]' '[:lower:]'))
$(eval AWS_SDK_VERSION := $(shell grep 'github.com/aws/aws-sdk-go' ../go.mod | cut -d ' ' -f 2))
echo Generating mock for AWS service $(AWS_SERVICE) and SDK version $(AWS_SDK_VERSION) && \
$(GOPATH)/bin/mockery --name $*API --output idktest/mocks --filename $(AWS_SERVICE).go --dir $(GOPATH)/pkg/mod/github.com/aws/aws-sdk-go@$(AWS_SDK_VERSION)/service/$(AWS_SERVICE)/$(AWS_SERVICE)iface
Reference in a new issue View git blame Copy permalink