diff --git a/.dockerignore b/.dockerignore
deleted file mode 100644
index 9abb766ce..000000000
--- a/.dockerignore
+++ /dev/null
@@ -1 +0,0 @@
-.*
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
index 1a4691b8c..715f81a6e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ env:
- secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA="
- secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o="
install:
- - make vendor generate-statik
+ - make install-dep install-statik vendor generate-statik
script:
- make test
# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 402895f1d..925f789b8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -66,6 +66,78 @@ If you want to help but you aren't sure where to start, check out our [github la
git remote add upstream git@github.com:pilosa/pilosa.git
```
+### Makefile
+
+Pilosa includes a Makefile that automates several tasks:
+
+- Install Pilosa:
+
+ ```sh
+ make install
+ ```
+
+- Install build dependencies (dep, statik, and protoc):
+
+ ```sh
+ make install-build-deps
+ ```
+
+- Create the vendor directory:
+
+ ```sh
+ make vendor
+ ```
+
+- Run the test suite:
+
+ ```sh
+ make test
+ ```
+
+- View the coverage report:
+
+ ```sh
+ make cover-viz
+ ```
+
+- Clear the `vendor/` and `build/` directories:
+
+ ```sh
+ make clean
+ ```
+
+- Create release tarballs:
+
+ ```sh
+ make release
+ ```
+
+- Generate static assets for the WebUI:
+
+ ```sh
+ make generate-statik
+ ```
+
+- Regenerate protocol buffer files in `internal/`:
+
+ ```sh
+ make generate-protoc
+ ```
+
+- Create tagged Docker image:
+
+ ```sh
+ make docker
+ ```
+
+- Run tests inside Docker container:
+
+ ```sh
+ make docker-test
+ ```
+
+Additional commands are available in the `Makefile`.
+
### Submitting code changes
- Before starting to work on a task, sync your branch with the upstream:
diff --git a/Dockerfile b/Dockerfile
index 50e88985c..bf076e526 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,9 @@
FROM golang:1.10 as builder
-ARG ldflags=''
-
-COPY . /go/src/github.com/pilosa/pilosa
+COPY . /go/src/github.com/pilosa/pilosa/
RUN cd /go/src/github.com/pilosa/pilosa \
- && make vendor \
- && CGO_ENABLED=0 go install -tags release -a -ldflags "$ldflags" github.com/pilosa/pilosa/cmd/pilosa
+ && CGO_ENABLED=0 make install-dep install-statik install FLAGS="-a"
FROM scratch
diff --git a/Makefile b/Makefile
index 8fdbcea3d..824d23cad 100644
--- a/Makefile
+++ b/Makefile
@@ -1,123 +1,134 @@
-.PHONY: dep docker pilosa release-build prerelease-build release prerelease prerelease-upload install generate generate-statik generate-protoc statik test cover cover-pkg cover-viz clean docker-build docker-test
+.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test generate generate-protoc generate-statik install install-build-deps install-dep install-protoc install-protoc-gen-gofast install-statik prerelease prerelease-build prerelease-upload release release-build require-dep require-protoc require-protoc-gen-gofast require-statik test
-DEP := $(shell command -v dep 2>/dev/null)
-STATIK := $(shell command -v statik 2>/dev/null)
-PROTOC := $(shell command -v protoc 2>/dev/null)
-VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
-STATUS := $(shell git status --porcelain)
-IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH)
CLONE_URL=github.com/pilosa/pilosa
-PKGS := $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor)
-BUILD_TIME=`date -u +%FT%T%z`
+VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
+VERSION_ID := $(VERSION)-$(GOOS)-$(GOARCH)
+BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
+BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
+BUILD_TIME := $(shell date -u +%FT%T%z)
LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME)"
-DOCKER_GOLANG_IMAGE=golang:latest
-GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
-BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(GIT_BRANCH))
-BRANCH_IDENTIFIER := $(BRANCH)-$(GOOS)-$(GOARCH)
+GO_VERSION=latest
-default: test pilosa
+# Run tests and compile Pilosa
+default: test build
+# Remove vendor and build directories
clean:
rm -rf vendor build
-$(GOPATH)/bin:
- mkdir $(GOPATH)/bin
-
-dep: $(GOPATH)/bin
- go get -u github.com/golang/dep/cmd/dep
-
+# Set up vendor directory using `dep`
vendor: Gopkg.toml
-ifndef DEP
- make dep
-endif
+ $(MAKE) require-dep
dep ensure
touch vendor
-Gopkg.lock: dep Gopkg.toml
- dep ensure
-
+# Run test suite
test: vendor
- go test $(PKGS) $(TESTFLAGS)
+ go test ./... $(TESTFLAGS)
+# Run test suite with coverage enabled
cover: vendor
- mkdir -p build/coverage
- echo "mode: set" > build/coverage/all.out
- for pkg in $(PKGS) ; do \
- make cover-pkg PKG=$$pkg ; \
- done
-
-cover-pkg:
- mkdir -p build/coverage
- touch build/coverage/$(subst /,-,$(PKG)).out
- go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG)
- tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out
+ mkdir -p build
+ $(MAKE) test TESTFLAGS="-coverprofile=build/coverage.out"
+# Run test suite with coverage enabled and view coverage results in browser
cover-viz: cover
- go tool cover -html=build/coverage/all.out
+ go tool cover -html=build/coverage.out
-pilosa: vendor
- go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+# Compile Pilosa
+build: vendor
+ go build -tags release -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
+# Create a single release build under the build directory
release-build: vendor
-ifdef DOCKER_BUILD
- make docker-build FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
-else
- make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
-endif
- cp LICENSE README.md build/pilosa-$(IDENTIFIER)
- tar -cvz -C build -f build/pilosa-$(IDENTIFIER).tar.gz pilosa-$(IDENTIFIER)/
- @echo "Created release build: build/pilosa-$(IDENTIFIER).tar.gz"
+ $(MAKE) $(if DOCKER_BUILD,docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa"
+ cp LICENSE README.md build/pilosa-$(VERSION_ID)
+ tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
+ @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz
-release:
-ifeq ($(STATUS),"")
- make release-build GOOS=darwin GOARCH=amd64
- make release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1
- make release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1
-else
- @echo "Will not create release with unclean git status."
-endif
+# Error out if there are untracked changes in Git
+check-clean:
+ $(if $(shell git status --porcelain),$(error Git status is not clean! Please commit or checkout/reset changes.))
+# Create release build tarballs for all supported platforms. Linux compilation happens under Docker.
+release: check-clean
+ $(MAKE) release-build GOOS=darwin GOARCH=amd64
+ $(MAKE) release-build GOOS=linux GOARCH=amd64 DOCKER_BUILD=1
+ $(MAKE) release-build GOOS=linux GOARCH=386 DOCKER_BUILD=1
+
+# Create branch-tagged pre-release for client library CI jobs
prerelease-build: vendor
- make pilosa FLAGS="-o build/pilosa-$(BRANCH_IDENTIFIER)/pilosa"
- cp LICENSE README.md build/pilosa-$(BRANCH_IDENTIFIER)
- tar -cvz -C build -f build/pilosa-$(BRANCH_IDENTIFIER).tar.gz pilosa-$(BRANCH_IDENTIFIER)/
- @echo "Created pre-release build: build/pilosa-$(BRANCH_IDENTIFIER).tar.gz"
+ $(MAKE) release-build VERSION_ID=$(BRANCH_ID)
+# Create prerelease build for Linux/amd64
prerelease:
- make prerelease-build GOOS=linux GOARCH=amd64
+ $(MAKE) prerelease-build GOOS=linux GOARCH=amd64
+# Upload prerelease to S3
prerelease-upload: prerelease
- aws s3 cp build/pilosa-$(BRANCH_IDENTIFIER).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_IDENTIFIER).tar.gz --acl public-read
+ aws s3 cp build/pilosa-$(BRANCH_ID).tar.gz s3://build.pilosa.com/pilosa-$(BRANCH_ID).tar.gz --acl public-read
+# Install Pilosa
install: vendor
- go install -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+ go install -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
-.protoc-gen-gofast: vendor
-ifndef PROTOC
- $(error "protoc is not available. please install protoc from https://github.com/google/protobuf/releases")
-endif
- go build -o .protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast
- cp ./.protoc-gen-gofast $(GOPATH)/bin/protoc-gen-gofast
-
-generate-protoc: .protoc-gen-gofast
+# `go generate` protocol buffers
+generate-protoc: require-protoc require-protoc-gen-gofast
go generate github.com/pilosa/pilosa/internal
-generate-statik: statik
+# `go generate` statik assets (WebUI)
+generate-statik: require-statik
go generate github.com/pilosa/pilosa/statik
+# `go generate` all needed packages
generate: generate-protoc generate-statik
-statik:
-ifndef STATIK
- go get github.com/rakyll/statik
-endif
-
+# Create Docker image from Dockerfile
docker:
- docker build -t "pilosa:$(VERSION)" --build-arg ldflags=$(LDFLAGS) .
- @echo "Created image: pilosa:$(VERSION)"
+ docker build -t "pilosa:$(VERSION)" .
+ @echo Created docker image: pilosa:$(VERSION)
+# Compile Pilosa inside Docker container
docker-build:
- docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) $(DOCKER_GOLANG_IMAGE) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+ docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) -e GOOS=$(GOOS) -e GOARCH=$(GOARCH) golang:$(GO_VERSION) go build -tags release -ldflags $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
+# Run Pilosa tests inside Docker container
docker-test:
- docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) $(DOCKER_GOLANG_IMAGE) go test $(TESTFLAGS) $(PKGS)
+ docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test $(TESTFLAGS) ./...
+
+######################
+# Build dependencies #
+######################
+
+# Verifies that needed build dependency is installed. Errors out if not installed.
+define require
+ $(if $(shell command -v $1 2>/dev/null),
+ $(info Verified build dependency "$1" is installed.),
+ $(error Build dependency "$1" not installed. To install, run `make install-$1` or `make install-build-deps`))
+endef
+
+require-dep:
+ $(call require,dep)
+
+require-statik:
+ $(call require,statik)
+
+require-protoc-gen-gofast:
+ $(call require,protoc-gen-gofast)
+
+require-protoc:
+ $(call require,protoc)
+
+install-build-deps: install-dep install-statik install-protoc-gen-gofast install-protoc
+
+install-dep:
+ go get -u github.com/golang/dep/cmd/dep
+
+install-statik:
+ go get -u github.com/rakyll/statik
+
+install-protoc-gen-gofast:
+ go get -u github.com/gogo/protobuf/protoc-gen-gofast
+
+install-protoc:
+ @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html
diff --git a/docs/installation.md b/docs/installation.md
index 9ca90ccf2..db0828b2f 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -134,6 +134,10 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
#### Build from Source
+
+
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
@@ -148,6 +152,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
```
cd $GOPATH/src/github.com/pilosa/pilosa
+ make install-build-deps
make generate-statik
make install
```
@@ -288,6 +293,10 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
#### Build from Source
+
+
1. Install the prerequisites:
* [Go](https://golang.org/doc/install). Be sure to set the `$GOPATH` and `$PATH` environment variables as described [here](https://golang.org/doc/code.html#GOPATH).
@@ -302,6 +311,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
3. Build the Pilosa repo (the `make generate-statik` line isn't necessary but builds a nice [webUI](../webui/) into Pilosa):
```
cd $GOPATH/src/github.com/pilosa/pilosa
+ make install-build-deps
make generate-statik
make install
```