From 99eb00bbdc7787c551996b14575fd981463a8435 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 23 Mar 2018 14:20:23 -0500 Subject: [PATCH 1/7] Remove obsolete coverage tools from Makefile Go 1.10 allows -coverprofile when testing multiple packages, so these hacks are no longer needed. --- Makefile | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 8fdbcea3d..9eb2df7e6 100644 --- a/Makefile +++ b/Makefile @@ -40,20 +40,10 @@ test: vendor go test $(PKGS) $(TESTFLAGS) cover: vendor - mkdir -p build/coverage - echo "mode: set" > build/coverage/all.out - for pkg in $(PKGS) ; do \ - make cover-pkg PKG=$$pkg ; \ - done - -cover-pkg: - mkdir -p build/coverage - touch build/coverage/$(subst /,-,$(PKG)).out - go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG) - tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out + make test TESTFLAGS="-coverprofile=build/coverage.out" cover-viz: cover - go tool cover -html=build/coverage/all.out + go tool cover -html=build/coverage.out pilosa: vendor go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa From 96e2ef534ff4e88e929562b9ed33560af46128c0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 27 Mar 2018 14:08:01 -0500 Subject: [PATCH 2/7] Clean up Makefile Turn off the automatic installation of build-time dependencies, which was confusing and inconsistent. Update PHONY list. Simplify git status check. Remove PKGS as Go 1.9+ automatically ignores `vendor` when running tests. Consolidate release/prerelease build code. Add comments. --- Makefile | 160 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 90 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index 9eb2df7e6..ddd6cb1a5 100644 --- a/Makefile +++ b/Makefile @@ -1,113 +1,133 @@ -.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate generate-statik generate-protoc statik test cover cover-pkg cover-viz clean docker-build docker-test +.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test -DEP := $(shell command -v dep 2>/dev/null) -STATIK := $(shell command -v statik 2>/dev/null) -PROTOC := $(shell command -v protoc 2>/dev/null) -VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) -STATUS := $(shell git status --porcelain) -IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) CLONE_URL=github.com/pilosa/pilosa -PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) -BUILD_TIME=`date -u +%FT%T%z` +VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) +VERSION_ID := $(VERSION)-$(GOOS)-$(GOARCH) +BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)) +BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) +BUILD_TIME := $(shell date -u +%FT%T%z) LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)" -DOCKER_GOLANG_IMAGE=golang:latest -GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) -BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(GIT_BRANCH)) -BRANCH_IDENTIFIER := $(BRANCH)-$(GOOS)-$(GOARCH) +GO_VERSION=latest -default: test pilosa +# Run tests and compile Pilosa +default: test build +# Remove vendor and build directories clean: rm -rf vendor build -$(GOPATH)/bin: - mkdir $(GOPATH)/bin - -dep: $(GOPATH)/bin - go get -u github.com/golang/dep/cmd/dep - +# Set up vendor directory using `dep` vendor: Gopkg.toml -ifndef DEP - make dep -endif + $(MAKE) require-dep dep ensure touch vendor -Gopkg.lock: dep Gopkg.toml - dep ensure - +# Run test suite test: vendor - go test $(PKGS) $(TESTFLAGS) + go test ./... $(TESTFLAGS) +# Run test suite with coverage enabled cover: vendor - make test TESTFLAGS="-coverprofile=build/coverage.out" + $(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out" +# Run test suite with coverage enabled and view coverage results in browser cover-viz: cover go tool cover -html=build/coverage.out -pilosa: vendor - go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa +# Compile Pilosa +build: vendor + go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +# Create a single release build under the build directory release-build: vendor -ifdef DOCKER_BUILD - make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" -else - make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" -endif - cp LICENSE README.md build/pilosa-$(IDENTIFIER) - tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/ - @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz" + $(MAKE) $(if DOCKER_BUILD,docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" + cp LICENSE README.md build/pilosa-$(VERSION_ID) + tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ + @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz -release: -ifeq ($(STATUS),"") - make release-build GOOS=darwin GOARCH=amd64 - make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 - make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 -else - @echo "Will not create release with unclean git status." -endif +# Error out if there are untracked changes in Git +check-clean: + $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.)) +# Create release build tarballs for all supported platforms. Linux compilation happens under Docker. +release: check-clean + $(MAKE) release-build GOOS=darwin GOARCH=amd64 + $(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1 + $(MAKE) release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1 + +# Create branch-tagged pre-release for client library CI jobs prerelease-build: vendor - make pilosa FLAGS="-o build/pilosa-$(BRANCH_IDENTIFIER)/pilosa" - cp LICENSE README.md build/pilosa-$(BRANCH_IDENTIFIER) - tar -cvz -C build -f build/pilosa-$(BRANCH_IDENTIFIER).tar.gz pilosa-$(BRANCH_IDENTIFIER)/ - @echo "Created pre-release build: build/pilosa-$(BRANCH_IDENTIFIER).tar.gz" + $(MAKE) release-build VERSION_ID=$(BRANCH_ID) +# Create prerelease build for Linux/amd64 prerelease: - make prerelease-build GOOS=linux GOARCH=amd64 + $(MAKE) prerelease-build GOOS=linux GOARCH=amd64 +# Upload prerelease to S3 prerelease-upload: prerelease - aws s3 cp build/pilosa-$(BRANCH_IDENTIFIER).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_IDENTIFIER).tar.gz --acl public-read + aws s3 cp build/pilosa-$(BRANCH_ID).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_ID).tar.gz --acl public-read +# Install Pilosa install: vendor - go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + go install -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa -.protoc-gen-gofast: vendor -ifndef PROTOC - $(error "protoc is not available. please install protoc from https://github.com/google/protobuf/releases") -endif - go build -o .protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast - cp ./.protoc-gen-gofast $(GOPATH)/bin/protoc-gen-gofast - -generate-protoc: .protoc-gen-gofast +# `go generate` protocol buffers +generate-protoc: require-protoc require-protoc-gen-gofast go generate github.com/pilosa/pilosa/internal -generate-statik: statik +# `go generate` statik assets (WebUI) +generate-statik: require-statik go generate github.com/pilosa/pilosa/statik +# `go generate` all needed packages generate: generate-protoc generate-statik -statik: -ifndef STATIK - go get github.com/rakyll/statik -endif - +# Create Docker image from Dockerfile docker: docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . - @echo "Created image: pilosa:$(VERSION)" + @echo Created docker image: pilosa:$(VERSION) +# Compile Pilosa inside Docker container docker-build: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa +# Run Pilosa tests inside Docker container docker-test: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS) + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(TESTFLAGS) ./... + +###################### +# Build dependencies # +###################### + +# Verifies that needed build dependency is installed. Errors out if not installed. +define require + $(if $(shell command -v $1 2>/dev/null), + $(info Verified build dependency "$1" is installed.), + $(error Build dependency "$1" not installed. To install, run `make install-$1` or `make install-build-deps`)) +endef + +require-dep: + $(call require,dep) + +require-statik: + $(call require,statik) + +require-protoc-gen-gofast: + $(call require,protoc-gen-gofast) + +require-protoc: + $(call require,protoc) + +install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc + +install-dep: + go get -u github.com/golang/dep/cmd/dep + +install-statik: + go get -u github.com/rakyll/statik + +install-protoc-gen-gofast: + go get -u github.com/gogo/protobuf/protoc-gen-gofast + +install-protoc: + @echo This tool cannot automatically install protoc. Please download and install protoc from https://github.com/google/protobuf/releases From e601d694c0e4bd7f1508471d00581815a6472379 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 27 Mar 2018 15:02:12 -0500 Subject: [PATCH 3/7] Document Makefile --- CONTRIBUTING.md | 66 ++++++++++++++++++++++++++++++++++++++++++++ docs/installation.md | 8 ++++++ 2 files changed, 74 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 402895f1d..0a29cb1b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,6 +66,72 @@ If you want to help but you aren't sure where to start, check out our [github la git remote add upstream git@github.com:pilosa/pilosa.git ``` +### Makefile + +Pilosa includes a Makefile that automates several tasks: + +- Install Pilosa: + + ```sh + make install + ``` + +- Create the vendor directory: + + ```sh + make vendor + ``` + +- Run the test suite: + + ```sh + make test + ``` + +- View the coverage report: + + ```sh + make cover-viz + ``` + +- Clear the `vendor/` and `build/` directories: + + ```sh + make clean + ``` + +- Create release tarballs: + + ```sh + make release + ``` + +- Generate static assets for the WebUI: + + ```sh + make generate-statik + ``` + +- Regenerate protocol buffer files in `internal/`: + + ```sh + make generate-protoc + ``` + +- Create tagged Docker image: + + ```sh + make docker + ``` + +- Run tests inside Docker container: + + ```sh + make docker-test + ``` + +Additional commands are available in the `Makefile`. + ### Submitting code changes - Before starting to work on a task, sync your branch with the upstream: diff --git a/docs/installation.md b/docs/installation.md index 9ca90ccf2..225e742ae 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -134,6 +134,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) #### Build from Source +
+

For advanced instructions for building from source, view our Contributor's Guide.

+
+ 1. Install the prerequisites: * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). @@ -288,6 +292,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende #### Build from Source +
+

For advanced instructions for building from source, view our Contributor's Guide.

+
+ 1. Install the prerequisites: * [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH). From 743726331fb893ffb7e23f68ba21195d91163253 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 28 Mar 2018 12:22:47 -0500 Subject: [PATCH 4/7] Add make instructions for travis-ci build process and use Makefile in Docker build --- .dockerignore | 1 - .travis.yml | 2 +- Dockerfile | 9 ++++----- Makefile | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 9abb766ce..000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 1a4691b8c..715f81a6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ env: - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" install: - - make vendor generate-statik + - make install-dep install-statik vendor generate-statik script: - make test # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 diff --git a/Dockerfile b/Dockerfile index 50e88985c..145405bc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,11 @@ FROM golang:1.10 as builder -ARG ldflags='' - -COPY . /go/src/github.com/pilosa/pilosa +COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ - && make vendor \ - && CGO_ENABLED=0 go install -tags release -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa + && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" + +RUN chmod +x /go/bin/pilosa FROM scratch diff --git a/Makefile b/Makefile index ddd6cb1a5..6094495e3 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ generate: generate-protoc generate-statik # Create Docker image from Dockerfile docker: - docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) . + docker build -t "pilosa:$(VERSION)" . @echo Created docker image: pilosa:$(VERSION) # Compile Pilosa inside Docker container From 9bf70705a3582e6b7c02af9c4dd3a77aa4c588b7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 28 Mar 2018 12:23:03 -0500 Subject: [PATCH 5/7] Add docs for "make install-build-deps" --- CONTRIBUTING.md | 6 ++++++ docs/installation.md | 2 ++ 2 files changed, 8 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a29cb1b5..925f789b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,6 +76,12 @@ Pilosa includes a Makefile that automates several tasks: make install ``` +- Install build dependencies (dep, statik, and protoc): + + ```sh + make install-build-deps + ``` + - Create the vendor directory: ```sh diff --git a/docs/installation.md b/docs/installation.md index 225e742ae..db0828b2f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -152,6 +152,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make install-build-deps make generate-statik make install ``` @@ -310,6 +311,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa): ``` cd $GOPATH/src/github.com/pilosa/pilosa + make install-build-deps make generate-statik make install ``` From d59d20d27bc86dff711e1a33134c005782aac9fb Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 29 Mar 2018 10:34:34 -0500 Subject: [PATCH 6/7] Redundant chmod --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 145405bc4..bf076e526 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,6 @@ COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a" -RUN chmod +x /go/bin/pilosa - FROM scratch LABEL maintainer "dev@pilosa.com" From ba9c26e6895a3fe05a331dcbef47bc64a41a18ff Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 2 Apr 2018 16:39:14 -0500 Subject: [PATCH 7/7] Code review fixes: Create build directory if it doesn't exist. Use better URL for protoc installation instructions. --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6094495e3..824d23cad 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ test: vendor # Run test suite with coverage enabled cover: vendor + mkdir -p build $(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out" # Run test suite with coverage enabled and view coverage results in browser @@ -130,4 +131,4 @@ install-protoc-gen-gofast: go get -u github.com/gogo/protobuf/protoc-gen-gofast install-protoc: - @echo This tool cannot automatically install protoc. Please download and install protoc from https://github.com/google/protobuf/releases + @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html