scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
featurebase/idk/Makefile
wmanongdo baab212bb1
[CLOUD-934] Optionally broadcast IDK Kinesis errors/panics to externa… (#2185) 
* [CLOUD-934] Optionally broadcast IDK Kinesis errors/panics to external storage

- Add a minor public method `idk.Main.SetLog` to allow setting the logger instance
  after initialization.
- Add a Logger implementation that captures recoverable errors and panics
  and pushes to an external store. Meant to decorate an existing Logger
  instance and always delegate to its implementation. Decoration happens
  when all AWS resources are initialized. Before then, the wrapped Logger
  implementation is used.
- If `--error-queue-name/CONSUMER_ERROR_QUEUE_NAME` specified, use an
  ErrorStreamLogger to push errors and panics to an SQS queue with that name.
  Omission of the option preserves current behavior.
- Parse sink ID from the `--stream-name/CONSUMER_STREAM_NAME` expecting the form
  'PREFIX'-VALID_UUID. If the sink UUID is invalid, emit a warning that errors/panics
  will not be written to an SQS queue but will still be logged using the decorated
  Logger instance.
- The inability to push to an SQS queue leads to warnings being emitted to notify
  ECS that no queue will be written to and is NOT a hard error.
- Add SQS interface mock for unit testing.
- Add IDK make targets for generating mock interfaces.

* [CLOUD-934] Execute go mod tidy and go fmt to pass CI/CD checks

* [CLOUD-934] Remove extraneous Makefile in idk/kinesis and fix install-mock-generator target

* [CLOUD-934] Add godocs to exported types and functions

* [CLOUD-934] Changed warning to not sound so ominous and update associated unit test

* [CLOUD-934] Unblock CI/CD at the IDK test stage
2022-08-12 17:59:56 -05:00

330 lines
14 KiB
Makefile
Raw Permalink Blame History

GO ?= go
GOOS ?= $(shell $(GO) env GOOS)
GOARCH ?= $(shell $(GO) env GOARCH)
GOPATH ?= $(shell $(GO) env GOPATH)
GO_VERSION=1.17.8
GO_BUILD_FLAGS=
ODBC_ENABLED=
BINOUT ?= bin
VERSION := $(shell git describe --tags 2> /dev/null || git rev-parse --verify --short=7 HEAD)
VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
BRANCH_NAME ?= ""
# We allow setting a custom docker-compose "project". Multiple of the
# same docker-compose environment can exist simultaneously as long as
# they use different projects (the project name is prepended to
# container names and such). This is useful in a CI environment where
# we might be running multiple instances of the tests concurrently.
PROJECT ?= idk
DOCKER_COMPOSE = docker-compose -p $(PROJECT)
UNAME_P := $(shell uname -p)
ifneq ($(filter arm%,$(UNAME_P)),)
IS_ARM := 1
else
IS_ARM := 0
endif
ifneq ($(filter Darwin%,$(shell uname)),)
IS_MAC := 1
else
IS_MAC := 0
endif
LDFLAGS="-X github.com/molecula/featurebase/idk.Version=$(VERSION) -X github.com/molecula/featurebase/idk.BuildTime=$(BUILD_TIME)"
LDFLAGS_STATIC="-linkmode external -extldflags \"-static\" -X github.com/molecula/featurebase/idk.Version=$(VERSION) -X github.com/molecula/featurebase/idk.BuildTime=$(BUILD_TIME) "
export GOPRIVATE=github.com/molecula
export CGO_ENABLED=1
# Cloud Deployment environment (dev, prod, staging, etc.), specified with the value of MCLOUD_ENV
# E.g. `MCLOUD_ENV=test make docker-push-ecr`
MCLOUD_ENV ?= dev
MCLOUD_ENV_FILE=.cloud-env.$(MCLOUD_ENV)
-include $(MCLOUD_ENV_FILE)
PROFILE ?=
AWS_REGION ?=
ECR_URL ?=
ECR_REPO ?= idk
BINS := $(patsubst cmd/%,$(BINOUT)/%,$(wildcard cmd/*))
.PHONY: install build release-build release test $(BINS)
install:
ifeq ($(filter $(IS_ARM),$(IS_MAC)), 1)
export HOMEBREW_NO_AUTO_UPDATE=1 && [ "${PKG_CONFIG_PATH}" != "$$(brew --prefix openssl)/lib/pkgconfig" ] && (brew list pkg-config || brew install pkg-config) && (brew list librdkafka || brew install librdkafka) && (brew list openssl || brew install openssl) && export PKG_CONFIG_PATH="$$(brew --prefix openssl)/lib/pkgconfig"; \
$(GO) install -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -tags dynamic ./cmd/...
else
$(GO) install -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) ./cmd/...
endif
$(BINS):
$(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o $@ ./cmd/$(notdir $@)
$(BINOUT)/molecula-consumer-sql-odbc:
CGO_ENABLED=1 $(GO) build -tags=odbc -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o $@ ./cmd/molecula-consumer-sql
build_non_cgo:
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/ingester ./cmd/ingester
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-csv ./cmd/molecula-consumer-csv
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-static ./cmd/molecula-consumer-kafka-static
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-sql ./cmd/molecula-consumer-sql
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-github ./cmd/molecula-consumer-github
CGO_ENABLED=0 $(GO) build -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kinesis ./cmd/molecula-consumer-kinesis
build_cgo:
ifeq ($(GOARCH), arm64)
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka ./cmd/molecula-consumer-kafka
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-delete ./cmd/molecula-consumer-kafka-delete
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/kafkagen ./cmd/kafkagen
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/kafkaput ./cmd/kafkaput
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/datagen ./cmd/datagen
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/bankgen ./cmd/bankgen
CGO_ENABLED=1 $(GO) build -tags dynamic $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-sasl ./cmd/molecula-consumer-kafka-sasl
endif
ifeq ($(GOARCH), amd64)
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka ./cmd/molecula-consumer-kafka
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-delete ./cmd/molecula-consumer-kafka-delete
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/kafkagen ./cmd/kafkagen
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/kafkaput ./cmd/kafkaput
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/datagen ./cmd/datagen
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/bankgen ./cmd/bankgen
CC=/usr/bin/musl-gcc CGO_ENABLED=1 $(GO) build -tags "musl static" -ldflags $(LDFLAGS_STATIC) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-kafka-sasl ./cmd/molecula-consumer-kafka-sasl
endif
CGO_ENABLED=1 $(GO) build -tags=odbc -ldflags $(LDFLAGS) $(GO_BUILD_FLAGS) -o bin/molecula-consumer-sql-odbc ./cmd/molecula-consumer-sql
BUILD_CGO ?= 0
build:
@echo GOOS=$(GOOS) GOARCH=$(GOARCH) uname -p=$(UNAME_P) build_cgo=$(BUILD_CGO)
make build_non_cgo
ifeq ($(BUILD_CGO), 1)
make build_cgo
endif
release-build:
mkdir -p build/molecula-consumer-$(VERSION_ID)/
$(MAKE) build BINOUT=build/molecula-consumer-$(VERSION_ID)/
tar -cvz -C build -f build/molecula-consumer-$(VERSION_ID).tar.gz molecula-consumer-$(VERSION_ID)/
@echo Created release build: build/molecula-consumer-$(VERSION_ID).tar.gz
release: check-clean
make release-build GOOS=linux GOARCH=amd64
make release-build GOOS=linux GOARCH=arm64
make release-build GOOS=darwin GOARCH=amd64
make release-build GOOS=darwin GOARCH=arm64
test:
$(GO) test ./...
test-local: testenv
IDK_TEST_CERT_PATH=$(shell pwd)/testenv/certs \
IDK_TEST_KAFKA_HOST=localhost:9092 \
IDK_TEST_PILOSA_GRPC_HOST=localhost:20101 \
IDK_TEST_PILOSA_HOST=localhost:10101 \
IDK_TEST_PILOSA_TLS_HOST=https://localhost:10111 \
IDK_TEST_REGISTRY_HOST=localhost:8081 \
$(GO) test $(TESTFLAGS) ./...
vendor: ../go.mod
$(GO) mod vendor
build-%:
$(DOCKER_COMPOSE) build $*
pull-%:
$(DOCKER_COMPOSE) pull $*
save-%-logs:
$(DOCKER_COMPOSE) logs $* > ./testdata/$(PROJECT)_$*_logs.txt
start-all: .pulled testenv build-wait
echo "branch name" ${BRANCH_NAME}
$(DOCKER_COMPOSE) up -d zookeeper
$(DOCKER_COMPOSE) run -T wait zookeeper 'echo "ruok" | nc -w 2 zookeeper 2181 | grep imok'
$(DOCKER_COMPOSE) up -d kafka
$(DOCKER_COMPOSE) up -d schema-registry
$(DOCKER_COMPOSE) up -d postgres
$(DOCKER_COMPOSE) run -T wait postgres pg_isready -h postgres -p 5432 -U postgres
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa-tls
BRANCH_NAME=${BRANCH_NAME} $(DOCKER_COMPOSE) up -d pilosa-auth
$(DOCKER_COMPOSE) run -T wait pilosa curl --silent --fail http://pilosa:10101/status
$(DOCKER_COMPOSE) run -T wait pilosa-tls curl --silent --cacert /certs/ca.crt --key /certs/theclient.key --cert /certs/theclient.crt --fail https://pilosa-tls:10111/status
$(DOCKER_COMPOSE) run -T wait pilosa-auth curl --silent --fail http://pilosa-auth:10105/version
$(DOCKER_COMPOSE) run -T wait kafka nc -z kafka 9092
$(DOCKER_COMPOSE) run -T wait schema-registry curl --silent --fail http://schema-registry:8081/config
start-postgres: build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d postgres
$(DOCKER_COMPOSE) run -T wait postgres pg_isready -h postgres -p 5432 -U postgres
start-pilosa: build-pilosa start-postgres build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d pilosa
$(DOCKER_COMPOSE) run -T wait pilosa curl --silent --fail http://pilosa:10101/status
start-pilosa-tls: build-pilosa-tls build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d pilosa-tls
$(DOCKER_COMPOSE) run -T wait pilosa-tls curl --silent --cacert /certs/ca.crt --key /certs/theclient.key --cert /certs/theclient.crt --fail https://pilosa-tls:10111/status
start-pilosa-auth: build-pilosa-auth build-wait testenv .pulled
$(DOCKER_COMPOSE) up -d pilosa-auth
$(DOCKER_COMPOSE) run -T wait pilosa-auth curl --silent --fail http://pilosa-auth:10105/version
start-zookeeper: pull-zookeeper testenv
$(DOCKER_COMPOSE) up -d zookeeper
$(DOCKER_COMPOSE) run -T wait zookeeper 'echo "ruok" | nc -w 2 zookeeper 2181 | grep imok'
start-kafka: pull-kafka start-zookeeper build-wait testenv
$(DOCKER_COMPOSE) up -d kafka
$(DOCKER_COMPOSE) run -T wait kafka nc -z kafka 9092
start-schema-registry: start-zookeeper start-kafka pull-schema-registry build-wait testenv
$(DOCKER_COMPOSE) up -d schema-registry
$(DOCKER_COMPOSE) run -T wait schema-registry curl --silent --fail http://schema-registry:8081/config
startup-old: start-postgres start-pilosa start-pilosa-tls start-pilosa-auth start-zookeeper start-kafka start-schema-registry
startup: start-all
shutdown:
$(DOCKER_COMPOSE) down -v --remove-orphans
rm -f .pulled
test-all: testenv
$(MAKE) startup
$(MAKE) test-run
$(DOCKER_COMPOSE) logs kafka | grep "Kafka version"
$(MAKE) shutdown
test-all-race: testenv
$(MAKE) startup
$(MAKE) test-run-race
$(MAKE) shutdown
test-all-kafka-sasl: testenv
$(MAKE) startup
$(MAKE) test-run-kafka-sasl
$(MAKE) shutdown
TCMD ?= ./...
# do "make startup", then e.g. "make test-run-local TCMD='-run=MyFavTest ./kafka'"
test-run-local:
pwd
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test go test -mod=vendor -tags=odbc,dynamic $(TCMD)
TPKG ?= ./...
test-run: testenv vendor
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -json -coverprofile=/testdata/$(PROJECT)_coverage.out | tee /testdata/$(PROJECT)_report.out"
$(DOCKER_COMPOSE) run -T idk-test /go/src/github.com/molecula/featurebase/idk/reingest_test.sh
test-run-race: testenv vendor
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -race -covermode=atomic -tags=dynamic $(TPKG) -coverpkg=$(TPKG) -timeout=30m -json -coverprofile=/testdata/$(PROJECT)_coverage.out | tee /testdata/$(PROJECT)_report.out"
test-run-kafka-sasl: testenv vendor
$(DOCKER_COMPOSE) build idk-test
$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v --tags=kafka_sasl -mod=vendor -race -timeout=30m $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -json -coverprofile=/testdata/$(PROJECT)_coverage.out | tee /testdata/$(PROJECT)_report.out"
.pulled:
$(DOCKER_COMPOSE) pull
touch .pulled
testenv: testenv/certs
testenv/certs:
certstrap --depot-path testenv/certs init --common-name ca --passphrase "" --expires "100 years"
certstrap --depot-path=testenv/certs request-cert --domain localhost --common-name localhost --passphrase ""
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" localhost
certstrap --depot-path=testenv/certs request-cert --domain pilosa-tls --common-name pilosa-tls --passphrase ""
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" pilosa-tls
certstrap --depot-path=testenv/certs request-cert --domain theclient --common-name theclient --passphrase ""
certstrap --depot-path=testenv/certs sign --CA ca --expires "100 years" theclient
# Create release using Docker
docker-release:
@if [ $(IS_ARM) = 1 ]; then \
echo Building linux-amd64 image on $(UNAME_P) is not supported; \
else \
$(MAKE) docker-build GOOS=linux GOARCH=amd64; \
fi
$(MAKE) docker-build GOOS=darwin GOARCH=amd64
$(MAKE) docker-build GOOS=darwin GOARCH=arm64
$(MAKE) docker-build GOOS=linux GOARCH=arm64
# This allows multiple concurrent builds to happen in CI without
# creating container name conflicts and such. (different BUILD_NAMEs
# are passed in from gitlab-ci.yml)
BUILD_NAME ?= idk-build
# Build a release in Docker
docker-build: vendor
DOCKER_BUILDKIT=0 docker build \
-f ../idk/Dockerfile \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
--build-arg GO_BUILD_FLAGS=$(GO_BUILD_FLAGS) \
--target builder \
--tag idk:$(BUILD_NAME) ../.
mkdir -p build/idk-$(GOOS)-$(GOARCH)
docker create --name $(BUILD_NAME) idk:$(BUILD_NAME)
docker cp $(BUILD_NAME):/featurebase/idk/bin/. ./build/idk-$(GOOS)-$(GOARCH)
docker rm $(BUILD_NAME)
# Create Docker image from Dockerfile
docker-image: vendor
docker build \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="GOOS=$(GOOS) GOARCH=$(GOARCH) BUILD_CGO=$(BUILD_CGO)" \
--tag registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID) .
@echo Created docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
docker: docker-image
docker-tag-push:
docker push registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
@echo Pushed docker image: registry.gitlab.com/molecula/featurebase/idk:$(VERSION_ID)
clean:
$(MAKE) shutdown
rm -rf testenv
# Error out if there are untracked changes in Git
check-clean:
ifndef SKIP_CHECK_CLEAN
$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
endif
docker-push-ecr: docker-image aws-login
docker tag idk:$(VERSION) $(ECR_URL)/$(ECR_REPO):$(VERSION)
aws ecr get-login-password --region $(AWS_REGION) --profile $(PROFILE)| docker login --username AWS --password-stdin $(ECR_URL)
docker push $(ECR_URL)/$(ECR_REPO):$(VERSION)
aws-login:
aws sso login --profile $(PROFILE)
# Build mock implementations of AWS service API interfaces for unit testing.
#
# These are run-once/rarely targets as they produce code artifacts only used
# during `go test`. An automated CI/CD process does NOT need to invoke every time.
MOCKS = S3 Kinesis SQS
install-mock-generator:
@which $(GOPATH)/bin/mockery || (echo "Installing missing dependency 'mockery' to generate mock implementations for unit testing." && \
$(GO) install 'github.com/vektra/mockery/v2@latest')
update-mocks: $(addprefix update-mocks-, $(MOCKS))
update-mocks-%: install-mock-generator
$(eval AWS_SERVICE := $(shell echo $* | tr '[:upper:]' '[:lower:]'))
$(eval AWS_SDK_VERSION := $(shell grep 'github.com/aws/aws-sdk-go' ../go.mod | cut -d ' ' -f 2))
echo Generating mock for AWS service $(AWS_SERVICE) and SDK version $(AWS_SDK_VERSION) && \
$(GOPATH)/bin/mockery --name $*API --output idktest/mocks --filename $(AWS_SERVICE).go --dir $(GOPATH)/pkg/mod/github.com/aws/aws-sdk-go@$(AWS_SDK_VERSION)/service/$(AWS_SERVICE)/$(AWS_SERVICE)iface
Reference in a new issue View git blame Copy permalink