diff --git a/.gitignore b/.gitignore index 8468c5367..7a41479b9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ vendor .DS_Store build *~ +lattice diff --git a/Makefile b/Makefile index bd605f1d3..93e5407f3 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) +LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null) VARIANT = Molecula VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH) BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) @@ -9,7 +10,7 @@ BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) -LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT)" +LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)" GO_VERSION=latest RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) @@ -135,10 +136,20 @@ prerelease-upload: install: go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +lattice: + git clone git@github.com:molecula/lattice.git + +build-lattice: lattice require-yarn + cd lattice && git pull && yarn install && yarn build + # `go generate` protocol buffers generate-protoc: require-protoc require-protoc-gen-gofast go generate github.com/pilosa/pilosa/v2/internal +# `go generate` statik assets (lattice UI) +generate-statik: build-lattice require-statik + go generate github.com/pilosa/pilosa/v2/statik + # `go generate` stringers generate-stringer: go generate github.com/pilosa/pilosa/v2 @@ -151,7 +162,7 @@ generate-proto-grpc: require-protoc require-protoc-gen-gofast protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto # `go generate` all needed packages -generate: generate-protoc generate-stringer generate-pql +generate: generate-protoc generate-statik generate-stringer generate-pql # Create Docker image from Dockerfile docker: vendor @@ -347,7 +358,10 @@ require-%: $(info Verified build dependency "$*" is installed.),\ $(error Build dependency "$*" not installed. To install, try `make install-$*`)) -install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg +install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg + +install-statik: + go get -u github.com/rakyll/statik install-stringer: GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer diff --git a/README.md b/README.md index 8057a469d..4543a3524 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst 1. [Install Pilosa](https://www.pilosa.com/docs/installation/). +Optionally, to include Lattice, the in-browser UI, follow the "Build from source" instructions, and run `make generate-statik` before `make install`. When you run a local Pilosa server on the default host, for example, you can access Lattice at [localhost:10101](http://localhost:10101). + 2. [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration: ```shell diff --git a/api.go b/api.go index 05808f61d..773c73ae4 100644 --- a/api.go +++ b/api.go @@ -1626,6 +1626,11 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } +// Version returns the Lattice version. +func (api *API) LatticeVersion() string { + return LatticeVersionInfo() +} + // Info returns information about this server instance. func (api *API) Info() serverInfo { si := api.server.systemInfo diff --git a/ctl/server.go b/ctl/server.go index 23a269612..a64b63749 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -39,7 +39,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { SetTLSConfig(flags, "", &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification) // Handler - flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/WebUI).") + flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/Lattice UI).") // Cluster flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)") diff --git a/filesystem.go b/filesystem.go new file mode 100644 index 000000000..62fc88d38 --- /dev/null +++ b/filesystem.go @@ -0,0 +1,42 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "fmt" + "net/http" +) + +// Ensure nopFileSystem implements interface. +var _ FileSystem = &nopFileSystem{} + +// FileSystem represents an interface for file system for serving the Lattice UI. +type FileSystem interface { + New() (http.FileSystem, error) +} + +func init() { + NopFileSystem = &nopFileSystem{} +} + +// NopFileSystem represents a FileSystem that returns an error if called. +var NopFileSystem FileSystem + +type nopFileSystem struct{} + +// New is a no-op implementation of FileSystem New method. +func (n *nopFileSystem) New() (http.FileSystem, error) { + return nil, fmt.Errorf("file system not implemented") +} diff --git a/go.mod b/go.mod index b8d683b60..a5e8fa41a 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.1.0 github.com/prometheus/prom2json v1.3.0 + github.com/rakyll/statik v0.1.7 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.18.12+incompatible diff --git a/go.sum b/go.sum index 5c5045296..47fd55364 100644 --- a/go.sum +++ b/go.sum @@ -167,6 +167,8 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y= github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM= +github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= +github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= diff --git a/holder.go b/holder.go index abe21b9ce..c531039a4 100644 --- a/holder.go +++ b/holder.go @@ -756,6 +756,7 @@ func (h *Holder) limitedSchema() []*IndexInfo { CreatedAt: index.CreatedAt(), Options: index.Options(), ShardWidth: ShardWidth, + Fields: make([]*FieldInfo, 0, len(index.Fields())), } for _, field := range index.Fields() { if strings.HasPrefix(field.name, "_") { diff --git a/http/handler.go b/http/handler.go index ea14e097c..c8953984c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -56,6 +56,8 @@ import ( type Handler struct { Handler http.Handler + fileSystem pilosa.FileSystem + logger logger.Logger // Keeps the query argument validators for each handler @@ -68,6 +70,8 @@ type Handler struct { closeTimeout time.Duration server *http.Server + + middleware []func(http.Handler) http.Handler } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -92,10 +96,10 @@ type handlerOption func(s *Handler) error func OptHandlerAllowedOrigins(origins []string) handlerOption { return func(h *Handler) error { - h.Handler = handlers.CORS( + h.middleware = append(h.middleware, handlers.CORS( handlers.AllowedOrigins(origins), handlers.AllowedHeaders([]string{"Content-Type"}), - )(h.Handler) + )) return nil } } @@ -107,6 +111,13 @@ func OptHandlerAPI(api *pilosa.API) handlerOption { } } +func OptHandlerFileSystem(fs pilosa.FileSystem) handlerOption { + return func(h *Handler) error { + h.fileSystem = fs + return nil + } +} + func OptHandlerLogger(logger logger.Logger) handlerOption { return func(h *Handler) error { h.logger = logger @@ -143,11 +154,10 @@ func NewHandler(opts ...handlerOption) (*Handler, error) { } }) handler := &Handler{ + fileSystem: pilosa.NopFileSystem, logger: logger.NopLogger, closeTimeout: time.Second * 30, } - handler.Handler = newRouter(handler) - handler.populateValidators() for _, opt := range opts { err := opt(handler) @@ -156,6 +166,10 @@ func NewHandler(opts ...handlerOption) (*Handler, error) { } } + // if OptHandlerFileSystem is used, it must be before newRouter is called + handler.Handler = newRouter(handler) + handler.populateValidators() + if handler.api == nil { return nil, errors.New("must pass OptHandlerAPI") } @@ -192,7 +206,6 @@ func (h *Handler) Close() error { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["Home"] = queryValidationSpecRequired() h.validators["PostClusterResizeAbort"] = queryValidationSpecRequired() h.validators["PostClusterResizeRemoveNode"] = queryValidationSpecRequired() h.validators["PostClusterResizeSetCoordinator"] = queryValidationSpecRequired() @@ -336,9 +349,8 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { } // newRouter creates a new mux http router. -func newRouter(handler *Handler) *mux.Router { +func newRouter(handler *Handler) http.Handler { router := mux.NewRouter() - router.HandleFunc("/", handler.handleHome).Methods("GET").Name("Home") router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort") router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator") @@ -400,11 +412,33 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.handlePostTranslateIndexDB).Methods("POST").Name("PostTranslateIndexDB") router.HandleFunc("/internal/translate/field/{index}/{field}", handler.handlePostTranslateFieldDB).Methods("POST").Name("PostTranslateFieldDB") + // Endpoints to support lattice UI embedded via statik. + // The messiness here reflects the fact that assets live in a nontrivial + // directory structure that is controlled externally. + latticeHandler := NewStatikHandler(handler) + router.PathPrefix("/static").Handler(latticeHandler) + router.Path("/").Handler(latticeHandler) + router.Path("/vds").Handler(latticeHandler) + router.Path("/favicon.png").Handler(latticeHandler) + router.Path("/favicon.svg").Handler(latticeHandler) + router.Path("/manifest.json").Handler(latticeHandler) + router.Use(handler.queryArgValidator) router.Use(handler.addQueryContext) router.Use(handler.extractTracing) router.Use(handler.collectStats) - return router + var h http.Handler = router + for _, middleware := range handler.middleware { + // Ideally, we would use `router.Use` to inject middleware, + // instead of wrapping the handler. The reason we can't is + // because the router will only apply middleware to matched + // handlers. In this case, it won't match handlers with the + // OPTIONS method, needed by the CORS middleware. This issue + // is described in detail here: + // https://github.com/gorilla/handlers/issues/142 + h = middleware(h) + } + return h } // ServeHTTP handles an HTTP request. @@ -422,6 +456,49 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.Handler.ServeHTTP(w, r) } +// statikHandler implements the http.Handler interface, and responds to +// requests for static assets with the appropriate file contents embedded +// in a statik filesystem. +type statikHandler struct { + handler *Handler + statikFS http.FileSystem +} + +// NewStatikHandler returns a new instance of statikHandler +func NewStatikHandler(h *Handler) statikHandler { + fs, err := h.fileSystem.New() + if err == nil { + h.logger.Printf("enabled Lattice UI (%s) at %s", h.api.LatticeVersion(), h.api.Node().URI) + } + + return statikHandler{ + handler: h, + statikFS: fs, + } +} + +func (s statikHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.UserAgent(), "curl") { + http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the Lattice UI by visiting this URL in your browser.", http.StatusNotFound) + return + } + + // /vds is a front-end route, not a backend route. Without this check, refreshing at /vds + // will request a nonexistent resource and return 404. + if r.URL.String() == "/vds" { + url, _ := url.Parse("/") + r.URL = url + } + + if s.statikFS == nil { + msg := "Lattice UI is not available. Please run `make generate-statik` before building Pilosa with `make install`." + s.handler.logger.Printf(msg) + http.Error(w, msg, http.StatusInternalServerError) + return + } + http.FileServer(s.statikFS).ServeHTTP(w, r) +} + // successResponse is a general success/error struct for http responses. type successResponse struct { h *Handler @@ -490,10 +567,6 @@ func (r *successResponse) write(w http.ResponseWriter, err error) { } } -func (h *Handler) handleHome(w http.ResponseWriter, _ *http.Request) { - http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound) -} - // validHeaderAcceptJSON returns false if one or more Accept // headers are present, but none of them are "application/json" // (or any matching wildcard). Otherwise returns true. diff --git a/server/server.go b/server/server.go index 546db5d41..bfd54cead 100644 --- a/server/server.go +++ b/server/server.go @@ -46,6 +46,7 @@ import ( "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/logger" "github.com/pilosa/pilosa/v2/prometheus" + "github.com/pilosa/pilosa/v2/statik" "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/statsd" "github.com/pilosa/pilosa/v2/syswrap" @@ -427,6 +428,7 @@ func (m *Command) SetupServer() error { http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), http.OptHandlerAPI(m.API), http.OptHandlerLogger(m.logger), + http.OptHandlerFileSystem(&statik.FileSystem{}), http.OptHandlerListener(m.ln), http.OptHandlerCloseTimeout(m.closeTimeout), ) diff --git a/statik/.gitignore b/statik/.gitignore new file mode 100644 index 000000000..485c0c57d --- /dev/null +++ b/statik/.gitignore @@ -0,0 +1 @@ +/statik.go diff --git a/statik/filesystem.go b/statik/filesystem.go new file mode 100644 index 000000000..c7f3141fc --- /dev/null +++ b/statik/filesystem.go @@ -0,0 +1,37 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//go:generate statik -src=../lattice/build -dest=../ +// +// Package statik contains static assets for the Lattice UI. `go generate` or +// `make generate-statik` will produce statik.go, which is ignored by git. +package statik + +import ( + "net/http" + + "github.com/pilosa/pilosa/v2" + "github.com/rakyll/statik/fs" +) + +// Ensure nopFileSystem implements interface. +var _ pilosa.FileSystem = &FileSystem{} + +// FileSystem represents a static FileSystem. +type FileSystem struct{} + +// New is a statik implementation of FileSystem New method. +func (s *FileSystem) New() (http.FileSystem, error) { + return fs.New() +} diff --git a/version.go b/version.go index 8671d683f..4313c9e33 100644 --- a/version.go +++ b/version.go @@ -20,6 +20,7 @@ var Version string var Commit string var Variant string var BuildTime string +var LatticeCommit string func VersionInfo() string { var prefix string @@ -50,3 +51,7 @@ func VersionInfo() string { return prefix + "Pilosa" + suffix } + +func LatticeVersionInfo() string { + return "g" + LatticeCommit +}