Merge branch 'master' into optimize-import

.gitignore

@@ -5,3 +5,4 @@ vendor
 .DS_Store
 build
 *~
+lattice

Makefile

@@ -1,7 +1,8 @@
-.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race
+.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race
 
 CLONE_URL=github.com/pilosa/pilosa
 VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
+LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null)
 VARIANT = Molecula
 VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH)
 BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
@@ -9,7 +10,7 @@ BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
 BUILD_TIME := $(shell date -u +%FT%T%z)
 SHARD_WIDTH = 20
 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD)
-LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT)"
+LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)"
 GO_VERSION=latest
 RELEASE ?= 0
 RELEASE_ENABLED = $(subst 0,,$(RELEASE))
@@ -135,10 +136,20 @@ prerelease-upload:
 install:
 	go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa
 
+lattice:
+	git clone git@github.com:molecula/lattice.git
+
+build-lattice: lattice require-yarn
+	cd lattice && git pull && yarn install && yarn build
+
 # `go generate` protocol buffers
 generate-protoc: require-protoc require-protoc-gen-gofast
 	go generate github.com/pilosa/pilosa/v2/internal
 
+# `go generate` statik assets (lattice UI)
+generate-statik: build-lattice require-statik
+	go generate github.com/pilosa/pilosa/v2/statik
+
 # `go generate` stringers
 generate-stringer:
 	go generate github.com/pilosa/pilosa/v2
@@ -151,7 +162,7 @@ generate-proto-grpc: require-protoc require-protoc-gen-gofast
 	protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto
 
 # `go generate` all needed packages
-generate: generate-protoc generate-stringer generate-pql
+generate: generate-protoc generate-statik generate-stringer generate-pql
 
 # Create Docker image from Dockerfile
 docker: vendor
@@ -347,7 +358,10 @@ require-%:
 		$(info Verified build dependency "$*" is installed.),\
 		$(error Build dependency "$*" not installed. To install, try `make install-$*`))
 
-install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg
+install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg
+
+install-statik:
+	go get -u github.com/rakyll/statik
 
 install-stringer:
 	GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer

README.md

@@ -31,6 +31,8 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst
 
 1.  [Install Pilosa](https://www.pilosa.com/docs/installation/).
 
+Optionally, to include Lattice, the in-browser UI, follow the "Build from source" instructions, and run `make generate-statik` before `make install`. When you run a local Pilosa server on the default host, for example, you can access Lattice at [localhost:10101](http://localhost:10101).
+
 2.  [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration:
 
     ```shell

api.go

@@ -1626,6 +1626,11 @@ func (api *API) Version() string {
 	return strings.TrimPrefix(Version, "v")
 }
 
+// Version returns the Lattice version.
+func (api *API) LatticeVersion() string {
+	return LatticeVersionInfo()
+}
+
 // Info returns information about this server instance.
 func (api *API) Info() serverInfo {
 	si := api.server.systemInfo

ctl/server.go

@@ -39,7 +39,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	SetTLSConfig(flags, "", &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification)
 
 	// Handler
-	flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/WebUI).")
+	flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/Lattice UI).")
 
 	// Cluster
 	flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)")

filesystem.go

@@ -0,0 +1,42 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa
+
+import (
+	"fmt"
+	"net/http"
+)
+
+// Ensure nopFileSystem implements interface.
+var _ FileSystem = &nopFileSystem{}
+
+// FileSystem represents an interface for file system for serving the Lattice UI.
+type FileSystem interface {
+	New() (http.FileSystem, error)
+}
+
+func init() {
+	NopFileSystem = &nopFileSystem{}
+}
+
+// NopFileSystem represents a FileSystem that returns an error if called.
+var NopFileSystem FileSystem
+
+type nopFileSystem struct{}
+
+// New is a no-op implementation of FileSystem New method.
+func (n *nopFileSystem) New() (http.FileSystem, error) {
+	return nil, fmt.Errorf("file system not implemented")
+}

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/prometheus/client_golang v1.0.0
 	github.com/prometheus/client_model v0.1.0
 	github.com/prometheus/prom2json v1.3.0
+	github.com/rakyll/statik v0.1.7
 	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
 	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil v2.18.12+incompatible

go.sum

@@ -167,6 +167,8 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=
 github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM=
+github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=

holder.go

@@ -756,6 +756,7 @@ func (h *Holder) limitedSchema() []*IndexInfo {
 			CreatedAt:  index.CreatedAt(),
 			Options:    index.Options(),
 			ShardWidth: ShardWidth,
+			Fields:     make([]*FieldInfo, 0, len(index.Fields())),
 		}
 		for _, field := range index.Fields() {
 			if strings.HasPrefix(field.name, "_") {

http/handler.go

@@ -56,6 +56,8 @@ import (
 type Handler struct {
 	Handler http.Handler
 
+	fileSystem pilosa.FileSystem
+
 	logger logger.Logger
 
 	// Keeps the query argument validators for each handler
@@ -68,6 +70,8 @@ type Handler struct {
 	closeTimeout time.Duration
 
 	server *http.Server
+
+	middleware []func(http.Handler) http.Handler
 }
 
 // externalPrefixFlag denotes endpoints that are intended to be exposed to clients.
@@ -92,10 +96,10 @@ type handlerOption func(s *Handler) error
 
 func OptHandlerAllowedOrigins(origins []string) handlerOption {
 	return func(h *Handler) error {
-		h.Handler = handlers.CORS(
+		h.middleware = append(h.middleware, handlers.CORS(
 			handlers.AllowedOrigins(origins),
 			handlers.AllowedHeaders([]string{"Content-Type"}),
-		)(h.Handler)
+		))
 		return nil
 	}
 }
@@ -107,6 +111,13 @@ func OptHandlerAPI(api *pilosa.API) handlerOption {
 	}
 }
 
+func OptHandlerFileSystem(fs pilosa.FileSystem) handlerOption {
+	return func(h *Handler) error {
+		h.fileSystem = fs
+		return nil
+	}
+}
+
 func OptHandlerLogger(logger logger.Logger) handlerOption {
 	return func(h *Handler) error {
 		h.logger = logger
@@ -143,11 +154,10 @@ func NewHandler(opts ...handlerOption) (*Handler, error) {
 		}
 	})
 	handler := &Handler{
+		fileSystem:   pilosa.NopFileSystem,
 		logger:       logger.NopLogger,
 		closeTimeout: time.Second * 30,
 	}
-	handler.Handler = newRouter(handler)
-	handler.populateValidators()
 
 	for _, opt := range opts {
 		err := opt(handler)
@@ -156,6 +166,10 @@ func NewHandler(opts ...handlerOption) (*Handler, error) {
 		}
 	}
 
+	// if OptHandlerFileSystem is used, it must be before newRouter is called
+	handler.Handler = newRouter(handler)
+	handler.populateValidators()
+
 	if handler.api == nil {
 		return nil, errors.New("must pass OptHandlerAPI")
 	}
@@ -192,7 +206,6 @@ func (h *Handler) Close() error {
 
 func (h *Handler) populateValidators() {
 	h.validators = map[string]*queryValidationSpec{}
-	h.validators["Home"] = queryValidationSpecRequired()
 	h.validators["PostClusterResizeAbort"] = queryValidationSpecRequired()
 	h.validators["PostClusterResizeRemoveNode"] = queryValidationSpecRequired()
 	h.validators["PostClusterResizeSetCoordinator"] = queryValidationSpecRequired()
@@ -336,9 +349,8 @@ func (h *Handler) collectStats(next http.Handler) http.Handler {
 }
 
 // newRouter creates a new mux http router.
-func newRouter(handler *Handler) *mux.Router {
+func newRouter(handler *Handler) http.Handler {
 	router := mux.NewRouter()
-	router.HandleFunc("/", handler.handleHome).Methods("GET").Name("Home")
 	router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort")
 	router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode")
 	router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator")
@@ -400,11 +412,33 @@ func newRouter(handler *Handler) *mux.Router {
 	router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.handlePostTranslateIndexDB).Methods("POST").Name("PostTranslateIndexDB")
 	router.HandleFunc("/internal/translate/field/{index}/{field}", handler.handlePostTranslateFieldDB).Methods("POST").Name("PostTranslateFieldDB")
 
+	// Endpoints to support lattice UI embedded via statik.
+	// The messiness here reflects the fact that assets live in a nontrivial
+	// directory structure that is controlled externally.
+	latticeHandler := NewStatikHandler(handler)
+	router.PathPrefix("/static").Handler(latticeHandler)
+	router.Path("/").Handler(latticeHandler)
+	router.Path("/vds").Handler(latticeHandler)
+	router.Path("/favicon.png").Handler(latticeHandler)
+	router.Path("/favicon.svg").Handler(latticeHandler)
+	router.Path("/manifest.json").Handler(latticeHandler)
+
 	router.Use(handler.queryArgValidator)
 	router.Use(handler.addQueryContext)
 	router.Use(handler.extractTracing)
 	router.Use(handler.collectStats)
-	return router
+	var h http.Handler = router
+	for _, middleware := range handler.middleware {
+		// Ideally, we would use `router.Use` to inject middleware,
+		// instead of wrapping the handler. The reason we can't is
+		// because the router will only apply middleware to matched
+		// handlers. In this case, it won't match handlers with the
+		// OPTIONS method, needed by the CORS middleware. This issue
+		// is described in detail here:
+		// https://github.com/gorilla/handlers/issues/142
+		h = middleware(h)
+	}
+	return h
 }
 
 // ServeHTTP handles an HTTP request.
@@ -422,6 +456,49 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.Handler.ServeHTTP(w, r)
 }
 
+// statikHandler implements the http.Handler interface, and responds to
+// requests for static assets with the appropriate file contents embedded
+// in a statik filesystem.
+type statikHandler struct {
+	handler  *Handler
+	statikFS http.FileSystem
+}
+
+// NewStatikHandler returns a new instance of statikHandler
+func NewStatikHandler(h *Handler) statikHandler {
+	fs, err := h.fileSystem.New()
+	if err == nil {
+		h.logger.Printf("enabled Lattice UI (%s) at %s", h.api.LatticeVersion(), h.api.Node().URI)
+	}
+
+	return statikHandler{
+		handler:  h,
+		statikFS: fs,
+	}
+}
+
+func (s statikHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if strings.HasPrefix(r.UserAgent(), "curl") {
+		http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the Lattice UI by visiting this URL in your browser.", http.StatusNotFound)
+		return
+	}
+
+	// /vds is a front-end route, not a backend route. Without this check, refreshing at /vds
+	// will request a nonexistent resource and return 404.
+	if r.URL.String() == "/vds" {
+		url, _ := url.Parse("/")
+		r.URL = url
+	}
+
+	if s.statikFS == nil {
+		msg := "Lattice UI is not available. Please run `make generate-statik` before building Pilosa with `make install`."
+		s.handler.logger.Printf(msg)
+		http.Error(w, msg, http.StatusInternalServerError)
+		return
+	}
+	http.FileServer(s.statikFS).ServeHTTP(w, r)
+}
+
 // successResponse is a general success/error struct for http responses.
 type successResponse struct {
 	h         *Handler
@@ -490,10 +567,6 @@ func (r *successResponse) write(w http.ResponseWriter, err error) {
 	}
 }
 
-func (h *Handler) handleHome(w http.ResponseWriter, _ *http.Request) {
-	http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound)
-}
-
 // validHeaderAcceptJSON returns false if one or more Accept
 // headers are present, but none of them are "application/json"
 // (or any matching wildcard). Otherwise returns true.

server/server.go

@@ -46,6 +46,7 @@ import (
 	"github.com/pilosa/pilosa/v2/http"
 	"github.com/pilosa/pilosa/v2/logger"
 	"github.com/pilosa/pilosa/v2/prometheus"
+	"github.com/pilosa/pilosa/v2/statik"
 	"github.com/pilosa/pilosa/v2/stats"
 	"github.com/pilosa/pilosa/v2/statsd"
 	"github.com/pilosa/pilosa/v2/syswrap"
@@ -427,6 +428,7 @@ func (m *Command) SetupServer() error {
 		http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins),
 		http.OptHandlerAPI(m.API),
 		http.OptHandlerLogger(m.logger),
+		http.OptHandlerFileSystem(&statik.FileSystem{}),
 		http.OptHandlerListener(m.ln),
 		http.OptHandlerCloseTimeout(m.closeTimeout),
 	)

statik/.gitignore

@@ -0,0 +1 @@
+/statik.go

statik/filesystem.go

@@ -0,0 +1,37 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+//go:generate statik -src=../lattice/build -dest=../
+//
+// Package statik contains static assets for the Lattice UI. `go generate` or
+// `make generate-statik` will produce statik.go, which is ignored by git.
+package statik
+
+import (
+	"net/http"
+
+	"github.com/pilosa/pilosa/v2"
+	"github.com/rakyll/statik/fs"
+)
+
+// Ensure nopFileSystem implements interface.
+var _ pilosa.FileSystem = &FileSystem{}
+
+// FileSystem represents a static FileSystem.
+type FileSystem struct{}
+
+// New is a statik implementation of FileSystem New method.
+func (s *FileSystem) New() (http.FileSystem, error) {
+	return fs.New()
+}

version.go

@@ -20,6 +20,7 @@ var Version string
 var Commit string
 var Variant string
 var BuildTime string
+var LatticeCommit string
 
 func VersionInfo() string {
 	var prefix string
@@ -50,3 +51,7 @@ func VersionInfo() string {
 
 	return prefix + "Pilosa" + suffix
 }
+
+func LatticeVersionInfo() string {
+	return "g" + LatticeCommit
+}