scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-06 16:45:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
featurebase/Makefile
Jason Aten ac7be132ef Tx integration milestone 
a) All tests green under -race for both PILOSA_TXSRC=roaring and PILOSA_TXSRC=badger.

b) Distinct is merged back into mainline pilosa.

Seebs notes on the Distinct work:

merge Distinct plugin back into main source tree, convert to Tx

We drop all references to the Preemptively Deprecated Don't You Dare
Use This extension interface, and move the one and only extension we had
(Distinct) into the main executor.

Also this fixes an arguable bug, which is that Container.AsBitmap()
would panic on a nil parameter, but it should have returned an empty
bitmap, because a nil *Ccontainer is a valid empty container. This
simplifies logic significantly in Distinct.

Fixes #569 #570 #571 #572 #573 #584 #585
2020-07-27 19:29:46 -04:00

295 lines
12 KiB
Makefile
Raw Blame History

.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test
CLONE_URL=github.com/pilosa/pilosa
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
VARIANT = Molecula
VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
BUILD_TIME := $(shell date -u +%FT%T%z)
SHARD_WIDTH = 20
COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT)"
GO_VERSION=latest
RELEASE ?= 0
RELEASE_ENABLED = $(subst 0,,$(RELEASE))
NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
define LICENSE_HASH_CODE
head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
endef
LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
export GO111MODULE=on
export GOPRIVATE=github.com/molecula
# Run tests and compile Pilosa
default: test build
# Remove build directories
clean:
rm -rf vendor build
# Set up vendor directory using `go mod vendor`
vendor: go.mod
go mod vendor
# Run test suite
test:
go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR)
# Run test suite with race flag
test-race:
go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v
bench:
go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)
# Run test suite with coverage enabled
cover:
mkdir -p build
$(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out"
# Run test suite with coverage enabled and view coverage results in browser
cover-viz: cover
go tool cover -html=build/coverage.out
# Compile Pilosa
build:
go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
# Create a single release build under the build directory
release-build:
$(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" RELEASE=1
cp NOTICE README.md LICENSE build/pilosa-$(VERSION_ID)
tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
@echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
# Error out if there are untracked changes in Git
check-clean:
ifndef SKIP_CHECK_CLEAN
$(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
endif
# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
release: check-clean
$(MAKE) release-build GOOS=darwin GOARCH=amd64
$(MAKE) release-build GOOS=linux GOARCH=amd64
$(MAKE) release-build GOOS=linux GOARCH=386
# try (e.g.) internal/clustertests/docker-compose-replication2.yml
DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
# Run cluster integration tests using docker. Requires docker daemon to be
# running. This will catch changes to internal/clustertests/*.go, but if you
# make changes to Pilosa, you'll want to run clustertests-build to rebuild the
# pilosa image.
clustertests: vendor
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) build client1
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
# Like clustertests, but rebuilds all images.
clustertests-build: vendor
docker-compose -f $(DOCKER_COMPOSE) down -v
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
# Create prerelease builds
prerelease:
$(MAKE) release-build GOOS=linux GOARCH=amd64 VERSION_ID=$$\(BRANCH_ID\)
$(if $(shell git describe --tags --exact-match HEAD),$(MAKE) release)
prerelease-upload:
aws s3 sync build/ s3://build.pilosa.com/ --exclude "*" --include "*.tar.gz" --acl public-read
# Install Pilosa
install:
go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
# `go generate` protocol buffers
generate-protoc: require-protoc require-protoc-gen-gofast
go generate github.com/pilosa/pilosa/v2/internal
# `go generate` stringers
generate-stringer:
go generate github.com/pilosa/pilosa/v2
generate-pql: require-peg
cd pql && peg -inline pql.peg && cd ..
# dunno if protoc-gen-gofast is actually needed here
generate-proto-grpc: require-protoc require-protoc-gen-gofast
protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
# `go generate` all needed packages
generate: generate-protoc generate-stringer generate-pql
# Create Docker image from Dockerfile
docker: vendor
docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" .
@echo Created docker image: pilosa:$(VERSION)
# Tag and push a Docker image
docker-tag-push: vendor
docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET)
docker push $(DOCKER_TARGET)
@echo Pushed docker image: $(DOCKER_TARGET)
# Compile Pilosa inside Docker container
docker-build:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
# Run Pilosa tests inside Docker container
docker-test:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
# run top tests, not subdirs. print summary red/green after.
# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
topt:
mv log.topt.roar log.topt.roar.prev || true
go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.roar
@echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
@echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l
topt-badger:
mv log.topt.badger log.topt.badger.prev || true
PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger
@echo " log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l
@echo " log.topt.badger red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
topt-rb:
mv log.topt.roaring_badger log.topt.roaring_badger.prev || true
PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger
@echo " log.topt.roaring_badger green: \c"; cat log.topt.roaring_badger | grep PASS |wc -l
@echo " log.topt.roaring_badger red: \c"; cat log.topt.roaring_badger | grep '\-\-\- FAIL' |wc -l
topt-badger-race:
mv log.topt.badger-race log.topt.badger-race.prev || true
PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger-race
@echo " log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l
@echo " log.topt.badger-race red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l
topt-rbf:
mv log.topt.rbf log.topt.rbf.prev || true
PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf
@echo " log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l
@echo " log.topt.rbf red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l
topt-race:
mv log.topt.race log.topt.race.prev || true
go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race
@echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
@echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l
# blue-green checks. These run two different storage engines (rbf, roaring, or badger)
# and compare each transaction for a result.
bg-br:
mv log.bg.bg_roar log.bg.bg_roar.prev || true
PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.bg_roar
@echo " log.bg.bg_roar green: \c"; cat log.bg.bg_roar | grep PASS |wc -l
@echo " log.bg.bg_roar red: \c"; cat log.bg.bg_roar | grep '\-\-\- FAIL' |wc -l
bg-rb:
mv log.bg.roar_bg log.bg.roar_bg.prev || true
PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.roar_bg
@echo " log.bg.roar_bg green: \c"; cat log.bg.roar_bg | grep PASS |wc -l
@echo " log.bg.roar_bg red: \c"; cat log.bg.roar_bg | grep '\-\-\- FAIL' |wc -l
bg-fr:
mv log.bg.rbf_roar log.bg.rbf_roar.prev || true
PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.rbf_roar
@echo " log.bg.rbf_roar green: \c"; cat log.bg.rbf_roar | grep PASS |wc -l
@echo " log.bg.rbf_roar red: \c"; cat log.bg.rbf_roar | grep '\-\-\- FAIL' |wc -l
bg-rf:
mv log.bg.roar_rbf log.bg.roar_rbf.prev || true
PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.roar_rbf
@echo " log.bg.roar_rbf green: \c"; cat log.bg.roar_rbf | grep PASS |wc -l
@echo " log.bg.roar_rbf red: \c"; cat log.bg.roar_rbf | grep '\-\-\- FAIL' |wc -l
bg-fb:
mv log.bg.rbf_badger log.bg.rbf_badger.prev || true
PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.rbf_badger
@echo " log.bg.rbf_badger green: \c"; cat log.bg.rbf_badger | grep PASS |wc -l
@echo " log.bg.rbf_badger red: \c"; cat log.bg.rbf_badger | grep '\-\-\- FAIL' |wc -l
bg-bf:
mv log.bg.badger_rbf log.bg.badger_rbf.prev || true
PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.badger_rbf
@echo " log.bg.badger_rbf green: \c"; cat log.bg.badger_rbf | grep PASS |wc -l
@echo " log.bg.badger_rbf red: \c"; cat log.bg.badger_rbf | grep '\-\-\- FAIL' |wc -l
# Run golangci-lint
golangci-lint: require-golangci-lint
golangci-lint run --skip-files '.*\.peg\.go'
# Alias
linter: golangci-lint
# Better alias
ocd: golangci-lint
# Run gometalinter with custom flags
gometalinter: require-gometalinter vendor
GO111MODULE=off gometalinter --vendor --disable-all \
--deadline=300s \
--enable=deadcode \
--enable=gochecknoinits \
--enable=gofmt \
--enable=goimports \
--enable=gotype \
--enable=gotypex \
--enable=ineffassign \
--enable=interfacer \
--enable=maligned \
--enable=misspell \
--enable=nakedret \
--enable=staticcheck \
--enable=unconvert \
--enable=unparam \
--enable=vet \
--exclude "^internal/.*\.pb\.go" \
--exclude "^pql/pql.peg.go" \
./...
# Verify that all Go files have license header
check-license-headers: SHELL:=/bin/bash
check-license-headers:
@! find . -path ./vendor -prune -o -name '*.go' -print | grep -v -F -f license.exceptions | while read fn;\
do [[ `$(call LICENSE_HASH_CODE, $$fn)` == $(LICENSE_HASH) ]] || echo $$fn; done | grep '.'
######################
# Build dependencies #
######################
# Verifies that needed build dependency is installed. Errors out if not installed.
require-%:
$(if $(shell command -v $* 2>/dev/null),\
$(info Verified build dependency "$*" is installed.),\
$(error Build dependency "$*" not installed. To install, try `make install-$*`))
install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg
install-stringer:
GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer
install-protoc-gen-gofast:
GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast
install-protoc:
@echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
install-peg:
GO111MODULE=off go get github.com/pointlander/peg
install-golangci-lint:
GO111MODULE=off go get github.com/golangci/golangci-lint/cmd/golangci-lint
install-gometalinter:
GO111MODULE=off go get -u github.com/alecthomas/gometalinter
GO111MODULE=off gometalinter --install
GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode
Reference in a new issue View git blame Copy permalink