scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 19:37:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
featurebase/idk/Makefile
souhailanoor 2bdf1f5081
FB-1597: unifying idk and featurebase (#2160) 
* unifying idk and featurebase: first pass

* resolved conflict with master for gitignore & dockerignore

* deleted binaries that were accidentally pushed to git

* combined gitlab jobs for idk & featurebase

* run go fmt for idk

* updated ssh env variable, and made docker password variable in gitlab env variables

* fixed typo assigning variable name

* trying to fix docker login error

* trying a different solution for docker password

* pass registry

* fixed docker login

* updated paths for idk

* exclude idk tests from featurebase test run

* fix vendor error

* update certificates

* grpc needs to be in version 1.38
genproto, which is imported by big query updates the grpc version to 1.47.0
grpc 1.47.0 causes etcd to deadlock when calling etcd.Close()
the fix is to have a replace in go.mod to specify a specific grpc version

* run go mod tidy

* go mod

* run go mod tidy

* exclude bigquery since it is causing issues and undo grpc replace in go.mod

* fix grpc version

* fix formatting error

* update formatting

* attempt to fix formatting

* update path for code coverage

* update to use current branch binaries, not master

* fix for building idk - path updates

* udpate path for binaries

* update job dependecies

* update docker idk tests to use the current branch registry

* update stages for jobs

* updated job dependencies

* not allow idk s3 dump to fail since it is a dependency for integration tests

* update dependecy for idk tests

* update paths for idk build and code coverage

* download featurebase binary from s3

* pass branch name to all setup scripts

* change to current branch instead of master

* updated sonarcloud

* sonarcloud fix and branch name fix

* trying to speed up pipeline run time

* update stage

* branch name fix + sonar cloud

* sonarcloud
2022-07-28 17:23:16 -05:00

311 lines
13 KiB
Makefile
Raw Blame History

GO ?= go
GOOS ?= $(shell $(GO) env GOOS)
GOARCH ?= $(shell $(GO) env GOARCH)
GO_VERSION=1.17.8
GO_BUILD_FLAGS=
ODBC_ENABLED=
BINOUT ?= bin
VERSION := $(shell git describe --tags 2> /dev/null || git rev-parse --verify --short=7 HEAD)
VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
BRANCH_NAME ?= ""
# We allow setting a custom docker-compose "project". Multiple of the
# same docker-compose environment can exist simultaneously as long as
# they use different projects (the project name is prepended to
# container names and such). This is useful in a CI environment where
# we might be running multiple instances of the tests concurrently.
PROJECT ?= idk
DOCKER_COMPOSE = docker-compose -p $(PROJECT)
UNAME_P := $(shell uname -p)
ifneq ($(filter arm%,$(UNAME_P)),)
IS_ARM := 1
else
IS_ARM := 0
endif
ifneq ($(filter Darwin%,$(shell uname)),)
IS_MAC := 1
else
IS_MAC := 0
endif
LDFLAGS="-X github.com/molecula/featurebase/idk.Version=$(VERSION) -X github.com/molecula/featurebase/idk.BuildTime=$(BUILD_TIME)"
LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X github.com/molecula/featurebase/idk.Version=$(VERSION) -X github.com/molecula/featurebase/idk.BuildTime=$(BUILD_TIME) "
export GOPRIVATE=github.com/molecula
export CGO_ENABLED=1
# Cloud Deployment environment (dev, prod, staging, etc.), specified with the value of MCLOUD_ENV
# E.g. `MCLOUD_ENV=test make docker-push-ecr`
MCLOUD_ENV ?= dev
MCLOUD_ENV_FILE=.cloud-env.$(MCLOUD_ENV)
-include $(MCLOUD_ENV_FILE)
PROFILE ?=
AWS_REGION ?=
ECR_URL ?=
ECR_REPO ?= idk
BINS := $(patsubst cmd/%,$(BINOUT)/%,$(wildcard cmd/*))
.PHONY: install build release-build release test $(BINS)
install:
ifeq ($(filter $(IS_ARM),$(IS_MAC)), 1)
export HOMEBREW_NO_AUTO_UPDATE=1 && [ "${PKG_CONFIG_PATH}" != "$$(brew --prefix openssl)/lib/pkgconfig" ] && (brew list pkg-config || brew install pkg-config) && (brew list librdkafka || brew install librdkafka) && (brew list openssl || brew install openssl) && export PKG_CONFIG_PATH="$$(brew --prefix openssl)/lib/pkgconfig"; \
$(GO) install -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -tags dynamic ./cmd/...
else
$(GO) install -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) ./cmd/...
endif
$(BINS):
$(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o $@ ./cmd/$(notdir $@)
$(BINOUT)/molecula-consumer-sql-odbc:
CGO_ENABLED=1 $(GO) build -tags=odbc -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o $@ ./cmd/molecula-consumer-sql
build_non_cgo:
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/ingester ./cmd/ingester
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-csv ./cmd/molecula-consumer-csv
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-static ./cmd/molecula-consumer-kafka-static
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-sql ./cmd/molecula-consumer-sql
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-github ./cmd/molecula-consumer-github
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kinesis ./cmd/molecula-consumer-kinesis
build_cgo:
ifeq ($(GOARCH), arm64)
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka ./cmd/molecula-consumer-kafka
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-delete ./cmd/molecula-consumer-kafka-delete
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/kafkagen ./cmd/kafkagen
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/kafkaput ./cmd/kafkaput
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/datagen ./cmd/datagen
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/bankgen ./cmd/bankgen
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-sasl ./cmd/molecula-consumer-kafka-sasl
endif
ifeq ($(GOARCH), amd64)
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka ./cmd/molecula-consumer-kafka
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-delete ./cmd/molecula-consumer-kafka-delete
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/kafkagen ./cmd/kafkagen
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/kafkaput ./cmd/kafkaput
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/datagen ./cmd/datagen
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/bankgen ./cmd/bankgen
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-sasl ./cmd/molecula-consumer-kafka-sasl
endif
CGO_ENABLED=1 $(GO) build -tags=odbc -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-sql-odbc ./cmd/molecula-consumer-sql
BUILD_CGO ?= 0
build:
@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
make build_non_cgo
ifeq ($(BUILD_CGO), 1)
make build_cgo
endif
release-build:
mkdir -p build/molecula-consumer-$(VERSION_ID)/
$(MAKE) build BINOUT=build/molecula-consumer-$(VERSION_ID)/
tar -cvz -C build -f build/molecula-consumer-$(VERSION_ID).tar.gz molecula-consumer-$(VERSION_ID)/
@echo Created release build: build/molecula-consumer-$(VERSION_ID).tar.gz
release: check-clean
make release-build GOOS=linux GOARCH=amd64
make release-build GOOS=linux GOARCH=arm64
make release-build GOOS=darwin GOARCH=amd64
make release-build GOOS=darwin GOARCH=arm64
test:
$(GO) test ./...
test-local: testenv
IDK_TEST_CERT_PATH=$(shell pwd)/testenv/certs \
IDK_TEST_KAFKA_HOST=localhost:9092 \
IDK_TEST_PILOSA_GRPC_HOST=localhost:20101 \
IDK_TEST_PILOSA_HOST=localhost:10101 \
IDK_TEST_PILOSA_TLS_HOST=https://localhost:10111 \
IDK_TEST_REGISTRY_HOST=localhost:8081 \
$(GO) test $(TESTFLAGS) ./...
vendor: ../go.mod
$(GO) mod vendor
build-%:
$(DOCKER_COMPOSE) build $*
pull-%:
$(DOCKER_COMPOSE) pull $*
save-%-logs:
$(DOCKER_COMPOSE) logs $* > ./testdata/$(PROJECT)_$*_logs.txt
start-all: .pulled testenv build-wait
echo "branch name" ${BRANCH_NAME}
$(DOCKER_COMPOSE) up -d zookeeper
$(DOCKER_COMPOSE) run -T wait zookeeper 'echo "ruok" | nc -w 2 zookeeper 2181 | grep imok'
$(DOCKER_COMPOSE) up -d kafka
$(DOCKER_COMPOSE) up -d schema-registry
$(DOCKER_COMPOSE) up -d postgres
$(DOCKER_COMPOSE) run -T wait postgres pg_isready -h postgres -p 5432 -U postgres
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa-tls
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa-auth
$(DOCKER_COMPOSE) run -T wait pilosa curl --silent --fail http://pilosa:10101/status
$(DOCKER_COMPOSE) run -T wait pilosa-tls curl --silent --cacert /certs/ca.crt --key /certs/theclient.key --cert /certs/theclient.crt --fail https://pilosa-tls:10111/status
$(DOCKER_COMPOSE) run -T wait pilosa-auth curl --silent --fail http://pilosa-auth:10105/version
$(DOCKER_COMPOSE) run -T wait kafka nc -z kafka 9092
$(DOCKER_COMPOSE) run -T wait schema-registry curl --silent --fail http://schema-registry:8081/config
start-postgres: build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d postgres
$(DOCKER_COMPOSE) run -T wait postgres pg_isready -h postgres -p 5432 -U postgres
start-pilosa: build-pilosa start-postgres build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d pilosa
$(DOCKER_COMPOSE) run -T wait pilosa curl --silent --fail http://pilosa:10101/status
start-pilosa-tls: build-pilosa-tls build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d pilosa-tls
$(DOCKER_COMPOSE) run -T wait pilosa-tls curl --silent --cacert /certs/ca.crt --key /certs/theclient.key --cert /certs/theclient.crt --fail https://pilosa-tls:10111/status
start-pilosa-auth: build-pilosa-auth build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d pilosa-auth
$(DOCKER_COMPOSE) run -T wait pilosa-auth curl --silent --fail http://pilosa-auth:10105/version
start-zookeeper: pull-zookeeper testenv
$(DOCKER_COMPOSE) up -d zookeeper
$(DOCKER_COMPOSE) run -T wait zookeeper 'echo "ruok" | nc -w 2 zookeeper 2181 | grep imok'
start-kafka: pull-kafka start-zookeeper build-wait testenv
$(DOCKER_COMPOSE) up -d kafka
$(DOCKER_COMPOSE) run -T wait kafka nc -z kafka 9092
start-schema-registry: start-zookeeper start-kafka pull-schema-registry build-wait testenv
$(DOCKER_COMPOSE) up -d schema-registry
$(DOCKER_COMPOSE) run -T wait schema-registry curl --silent --fail http://schema-registry:8081/config
startup-old: start-postgres start-pilosa start-pilosa-tls start-pilosa-auth start-zookeeper start-kafka start-schema-registry
startup: start-all
shutdown:
$(DOCKER_COMPOSE) down -v --remove-orphans
rm .pulled
test-all: testenv
$(MAKE) startup
$(MAKE) test-run
$(DOCKER_COMPOSE) logs kafka | grep "Kafka version"
$(MAKE) shutdown
test-all-race: testenv
$(MAKE) startup
$(MAKE) test-run-race
$(MAKE) shutdown
test-all-kafka-sasl: testenv
$(MAKE) startup
$(MAKE) test-run-kafka-sasl
$(MAKE) shutdown
TCMD ?= ./...
# do "make startup", then e.g. "make test-run-local TCMD='-run=MyFavTest ./kafka'"
test-run-local:
pwd
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test go test -mod=vendor -tags=odbc,dynamic $(TCMD)
TPKG ?= ./...
test-run: testenv vendor
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -json -coverprofile=/testdata/$(PROJECT)_coverage.out | tee /testdata/$(PROJECT)_report.out"
$(DOCKER_COMPOSE) run -T idk-test /go/src/github.com/molecula/featurebase/idk/reingest_test.sh
test-run-race: testenv vendor
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -race -covermode=atomic -tags=dynamic $(TPKG) -coverpkg=$(TPKG) -timeout=30m -json -coverprofile=/testdata/$(PROJECT)_coverage.out | tee /testdata/$(PROJECT)_report.out"
test-run-kafka-sasl: testenv vendor
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v --tags=kafka_sasl -mod=vendor -race -timeout=30m $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -json -coverprofile=/testdata/$(PROJECT)_coverage.out | tee /testdata/$(PROJECT)_report.out"
.pulled:
$(DOCKER_COMPOSE) pull
touch .pulled
testenv: testenv/certs
testenv/certs:
certstrap --depot-path testenv/certs init --common-name ca --passphrase "" --expires "100 years"
certstrap --depot-path=testenv/certs request-cert --domain localhost --common-name localhost --passphrase ""
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" localhost
certstrap --depot-path=testenv/certs request-cert --domain pilosa-tls --common-name pilosa-tls --passphrase ""
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" pilosa-tls
certstrap --depot-path=testenv/certs request-cert --domain theclient --common-name theclient --passphrase ""
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" theclient
# Create release using Docker
docker-release:
@if [ $(IS_ARM) = 1 ]; then \
echo Building linux-amd64 image on $(UNAME_P) is not supported; \
else \
$(MAKE) docker-build GOOS=linux GOARCH=amd64; \
fi
$(MAKE) docker-build GOOS=darwin GOARCH=amd64
$(MAKE) docker-build GOOS=darwin GOARCH=arm64
$(MAKE) docker-build GOOS=linux GOARCH=arm64
# This allows multiple concurrent builds to happen in CI without
# creating container name conflicts and such. (different BUILD_NAMEs
# are passed in from gitlab-ci.yml)
BUILD_NAME ?= idk-build
# Build a release in Docker
docker-build: vendor
DOCKER_BUILDKIT=0 docker build \
-f ../idk/Dockerfile \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
--build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
--target builder \
--tag idk:$(BUILD_NAME) ../.
mkdir -p build/idk-$(GOOS)-$(GOARCH)
docker create --name $(BUILD_NAME) idk:$(BUILD_NAME)
docker cp $(BUILD_NAME):/featurebase/idk/bin/. ./build/idk-$(GOOS)-$(GOARCH)
docker rm $(BUILD_NAME)
# Create Docker image from Dockerfile
docker-image: vendor
docker build \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
--tag registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) .
@echo Created docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
docker: docker-image
docker-tag-push:
docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
@echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
clean:
$(MAKE) shutdown
rm -rf testenv
# Error out if there are untracked changes in Git
check-clean:
ifndef SKIP_CHECK_CLEAN
$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
endif
docker-push-ecr: docker-image aws-login
docker tag idk:$(VERSION) $(ECR_URL)/$(ECR_REPO):$(VERSION)
aws ecr get-login-password --region $(AWS_REGION) --profile $(PROFILE)| docker login --username AWS --password-stdin $(ECR_URL)
docker push $(ECR_URL)/$(ECR_REPO):$(VERSION)
aws-login:
aws sso login --profile $(PROFILE)
Reference in a new issue View git blame Copy permalink