From 88a288e77522fa6f9e7c0eb7c0c082ba30324ea7 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 25 Aug 2020 19:34:29 -0500 Subject: [PATCH 01/11] Embed lattice via statik --- Makefile | 17 ++++++++++++++--- go.mod | 1 + go.sum | 2 ++ http/handler.go | 34 +++++++++++++++++++++++++++++----- server/server.go | 2 ++ statik/.gitignore | 1 + statik/filesystem.go | 37 +++++++++++++++++++++++++++++++++++++ 7 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 statik/.gitignore create mode 100644 statik/filesystem.go diff --git a/Makefile b/Makefile index bd605f1d3..6b7ccfaf5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build require-statik test testv testv-race testvsub testvsub-race CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -135,10 +135,18 @@ prerelease-upload: install: go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +build-lattice: require-yarn + git clone git@github.com:molecula/lattice.git + cd lattice && yarn install && yarn build + # `go generate` protocol buffers generate-protoc: require-protoc require-protoc-gen-gofast go generate github.com/pilosa/pilosa/v2/internal +# `go generate` statik assets (lattice UI) +generate-statik: build-lattice require-statik + go generate github.com/pilosa/pilosa/v2/statik + # `go generate` stringers generate-stringer: go generate github.com/pilosa/pilosa/v2 @@ -151,7 +159,7 @@ generate-proto-grpc: require-protoc require-protoc-gen-gofast protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto # `go generate` all needed packages -generate: generate-protoc generate-stringer generate-pql +generate: generate-protoc generate-statik generate-stringer generate-pql # Create Docker image from Dockerfile docker: vendor @@ -347,7 +355,10 @@ require-%: $(info Verified build dependency "$*" is installed.),\ $(error Build dependency "$*" not installed. To install, try `make install-$*`)) -install-build-deps: install-protoc-gen-gofast install-protoc install-stringer install-peg +install-build-deps: install-protoc-gen-gofast install-protoc install-statik install-stringer install-peg + +install-statik: + go get -u github.com/rakyll/statik install-stringer: GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer diff --git a/go.mod b/go.mod index b8d683b60..a5e8fa41a 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_model v0.1.0 github.com/prometheus/prom2json v1.3.0 + github.com/rakyll/statik v0.1.7 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.18.12+incompatible diff --git a/go.sum b/go.sum index 5c5045296..47fd55364 100644 --- a/go.sum +++ b/go.sum @@ -167,6 +167,8 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y= github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM= +github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= +github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= diff --git a/http/handler.go b/http/handler.go index ea14e097c..1aff0af04 100644 --- a/http/handler.go +++ b/http/handler.go @@ -56,6 +56,8 @@ import ( type Handler struct { Handler http.Handler + FileSystem pilosa.FileSystem + logger logger.Logger // Keeps the query argument validators for each handler @@ -107,6 +109,13 @@ func OptHandlerAPI(api *pilosa.API) handlerOption { } } +func OptHandlerFileSystem(fs pilosa.FileSystem) handlerOption { + return func(h *Handler) error { + h.FileSystem = fs + return nil + } +} + func OptHandlerLogger(logger logger.Logger) handlerOption { return func(h *Handler) error { h.logger = logger @@ -143,6 +152,7 @@ func NewHandler(opts ...handlerOption) (*Handler, error) { } }) handler := &Handler{ + FileSystem: pilosa.NopFileSystem, logger: logger.NopLogger, closeTimeout: time.Second * 30, } @@ -338,7 +348,9 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { // newRouter creates a new mux http router. func newRouter(handler *Handler) *mux.Router { router := mux.NewRouter() - router.HandleFunc("/", handler.handleHome).Methods("GET").Name("Home") + router.HandleFunc("/", handler.handleLattice).Methods("GET") + router.HandleFunc("/{file}", handler.handleLattice).Methods("GET") + router.HandleFunc("/static/{file}", handler.handleLattice).Methods("GET") router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort") router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator") @@ -422,6 +434,22 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.Handler.ServeHTTP(w, r) } +func (h *Handler) handleLattice(w http.ResponseWriter, r *http.Request) { + // If user is using curl, don't chuck HTML at them + if strings.HasPrefix(r.UserAgent(), "curl") { + http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the Lattice UI by visiting this URL in your browser.", http.StatusNotFound) + return + } + filesystem, err := h.FileSystem.New() + + if err != nil { + _ = h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) + h.logger.Printf("Lattice UI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") + return + } + http.FileServer(filesystem).ServeHTTP(w, r) +} + // successResponse is a general success/error struct for http responses. type successResponse struct { h *Handler @@ -490,10 +518,6 @@ func (r *successResponse) write(w http.ResponseWriter, err error) { } } -func (h *Handler) handleHome(w http.ResponseWriter, _ *http.Request) { - http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information.", http.StatusNotFound) -} - // validHeaderAcceptJSON returns false if one or more Accept // headers are present, but none of them are "application/json" // (or any matching wildcard). Otherwise returns true. diff --git a/server/server.go b/server/server.go index 546db5d41..bfd54cead 100644 --- a/server/server.go +++ b/server/server.go @@ -46,6 +46,7 @@ import ( "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/logger" "github.com/pilosa/pilosa/v2/prometheus" + "github.com/pilosa/pilosa/v2/statik" "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/statsd" "github.com/pilosa/pilosa/v2/syswrap" @@ -427,6 +428,7 @@ func (m *Command) SetupServer() error { http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins), http.OptHandlerAPI(m.API), http.OptHandlerLogger(m.logger), + http.OptHandlerFileSystem(&statik.FileSystem{}), http.OptHandlerListener(m.ln), http.OptHandlerCloseTimeout(m.closeTimeout), ) diff --git a/statik/.gitignore b/statik/.gitignore new file mode 100644 index 000000000..485c0c57d --- /dev/null +++ b/statik/.gitignore @@ -0,0 +1 @@ +/statik.go diff --git a/statik/filesystem.go b/statik/filesystem.go new file mode 100644 index 000000000..a5ec437b5 --- /dev/null +++ b/statik/filesystem.go @@ -0,0 +1,37 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//go:generate statik -src=../lattice/build -dest=../ +// +// Package statik contains static assets for the Web UI. `go generate` or +// `make generate-statik` will produce statik.go, which is ignored by git. +package statik + +import ( + "net/http" + + "github.com/pilosa/pilosa/v2" + "github.com/rakyll/statik/fs" +) + +// Ensure nopFileSystem implements interface. +var _ pilosa.FileSystem = &FileSystem{} + +// FileSystem represents a static FileSystem. +type FileSystem struct{} + +// New is a statik implementation of FileSystem New method. +func (s *FileSystem) New() (http.FileSystem, error) { + return fs.New() +} From d912403eb2ef3ace35b0ad7f8e8f565b144fbb2c Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 25 Aug 2020 19:43:14 -0500 Subject: [PATCH 02/11] Add missing file --- ctl/server.go | 2 +- filesystem.go | 42 ++++++++++++++++++++++++++++++++++++++++++ statik/filesystem.go | 2 +- 3 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 filesystem.go diff --git a/ctl/server.go b/ctl/server.go index 23a269612..a64b63749 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -39,7 +39,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { SetTLSConfig(flags, "", &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.CACertPath, &srv.Config.TLS.SkipVerify, &srv.Config.TLS.EnableClientVerification) // Handler - flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/WebUI).") + flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/Lattice UI).") // Cluster flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)") diff --git a/filesystem.go b/filesystem.go new file mode 100644 index 000000000..62fc88d38 --- /dev/null +++ b/filesystem.go @@ -0,0 +1,42 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "fmt" + "net/http" +) + +// Ensure nopFileSystem implements interface. +var _ FileSystem = &nopFileSystem{} + +// FileSystem represents an interface for file system for serving the Lattice UI. +type FileSystem interface { + New() (http.FileSystem, error) +} + +func init() { + NopFileSystem = &nopFileSystem{} +} + +// NopFileSystem represents a FileSystem that returns an error if called. +var NopFileSystem FileSystem + +type nopFileSystem struct{} + +// New is a no-op implementation of FileSystem New method. +func (n *nopFileSystem) New() (http.FileSystem, error) { + return nil, fmt.Errorf("file system not implemented") +} diff --git a/statik/filesystem.go b/statik/filesystem.go index a5ec437b5..c7f3141fc 100644 --- a/statik/filesystem.go +++ b/statik/filesystem.go @@ -14,7 +14,7 @@ // //go:generate statik -src=../lattice/build -dest=../ // -// Package statik contains static assets for the Web UI. `go generate` or +// Package statik contains static assets for the Lattice UI. `go generate` or // `make generate-statik` will produce statik.go, which is ignored by git. package statik From 19ee27fdb6e79e4519336d9360006f5786da76ae Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 27 Aug 2020 14:41:04 -0500 Subject: [PATCH 03/11] Use SPA handler to serve from filesystem, to test routing behavior --- http/handler.go | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/http/handler.go b/http/handler.go index 1aff0af04..4a0771292 100644 --- a/http/handler.go +++ b/http/handler.go @@ -29,6 +29,7 @@ import ( _ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server. "net/url" "os" + "path/filepath" "reflect" "runtime/debug" "runtime/pprof" @@ -345,12 +346,43 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { }) } +// latticeHandler implements the http.Handler interface, so we can use it +// to respond to HTTP requests. The path to the static directory and +// path to the index file within that static directory are used to +// serve the Lattice UI in the given static directory +type latticeHandler struct { + staticPath string + indexPath string +} + +// ServeHTTP inspects the URL path to locate a file within the static dir +// on latticeHandler. If a file is found, it will be served. If not, the +// file located at the index path on the latticeHandler will be served. This +// is suitable behavior for serving an SPA. +func (h latticeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // get the absolute path to prevent directory traversal + path, err := filepath.Abs(r.URL.Path) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) // TODO + return + } + + path = filepath.Join(h.staticPath, path) + _, err = os.Stat(path) // TODO + if os.IsNotExist(err) { + http.ServeFile(w, r, filepath.Join(h.staticPath, h.indexPath)) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + http.FileServer(http.Dir(h.staticPath)).ServeHTTP(w, r) +} + // newRouter creates a new mux http router. func newRouter(handler *Handler) *mux.Router { router := mux.NewRouter() - router.HandleFunc("/", handler.handleLattice).Methods("GET") - router.HandleFunc("/{file}", handler.handleLattice).Methods("GET") - router.HandleFunc("/static/{file}", handler.handleLattice).Methods("GET") router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort") router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST").Name("PostClusterResizeSetCoordinator") @@ -412,6 +444,9 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.handlePostTranslateIndexDB).Methods("POST").Name("PostTranslateIndexDB") router.HandleFunc("/internal/translate/field/{index}/{field}", handler.handlePostTranslateFieldDB).Methods("POST").Name("PostTranslateFieldDB") + lattice := latticeHandler{staticPath: "lattice/build", indexPath: "index.html"} + router.PathPrefix("/").Handler(lattice) + router.Use(handler.queryArgValidator) router.Use(handler.addQueryContext) router.Use(handler.extractTracing) From b039bd50a1ca2eaa95ada6b41bd58a824808477f Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 27 Aug 2020 17:44:26 -0500 Subject: [PATCH 04/11] Switch to mux PathPrefix matcher entirely --- http/handler.go | 105 ++++++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/http/handler.go b/http/handler.go index 4a0771292..b1981c901 100644 --- a/http/handler.go +++ b/http/handler.go @@ -29,7 +29,6 @@ import ( _ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server. "net/url" "os" - "path/filepath" "reflect" "runtime/debug" "runtime/pprof" @@ -157,8 +156,6 @@ func NewHandler(opts ...handlerOption) (*Handler, error) { logger: logger.NopLogger, closeTimeout: time.Second * 30, } - handler.Handler = newRouter(handler) - handler.populateValidators() for _, opt := range opts { err := opt(handler) @@ -167,6 +164,10 @@ func NewHandler(opts ...handlerOption) (*Handler, error) { } } + // if OptHandlerFileSystem is used, it must be before newRouter is called + handler.Handler = newRouter(handler) + handler.populateValidators() + if handler.api == nil { return nil, errors.New("must pass OptHandlerAPI") } @@ -203,7 +204,6 @@ func (h *Handler) Close() error { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["Home"] = queryValidationSpecRequired() h.validators["PostClusterResizeAbort"] = queryValidationSpecRequired() h.validators["PostClusterResizeRemoveNode"] = queryValidationSpecRequired() h.validators["PostClusterResizeSetCoordinator"] = queryValidationSpecRequired() @@ -346,40 +346,6 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { }) } -// latticeHandler implements the http.Handler interface, so we can use it -// to respond to HTTP requests. The path to the static directory and -// path to the index file within that static directory are used to -// serve the Lattice UI in the given static directory -type latticeHandler struct { - staticPath string - indexPath string -} - -// ServeHTTP inspects the URL path to locate a file within the static dir -// on latticeHandler. If a file is found, it will be served. If not, the -// file located at the index path on the latticeHandler will be served. This -// is suitable behavior for serving an SPA. -func (h latticeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // get the absolute path to prevent directory traversal - path, err := filepath.Abs(r.URL.Path) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) // TODO - return - } - - path = filepath.Join(h.staticPath, path) - _, err = os.Stat(path) // TODO - if os.IsNotExist(err) { - http.ServeFile(w, r, filepath.Join(h.staticPath, h.indexPath)) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - http.FileServer(http.Dir(h.staticPath)).ServeHTTP(w, r) -} - // newRouter creates a new mux http router. func newRouter(handler *Handler) *mux.Router { router := mux.NewRouter() @@ -440,12 +406,19 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/index/{index}/field/{field}/remote-available-shards/{shardID}", handler.handleDeleteRemoteAvailableShard).Methods("DELETE") router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.handlePostTranslateIndexDB).Methods("POST").Name("PostTranslateIndexDB") router.HandleFunc("/internal/translate/field/{index}/{field}", handler.handlePostTranslateFieldDB).Methods("POST").Name("PostTranslateFieldDB") - lattice := latticeHandler{staticPath: "lattice/build", indexPath: "index.html"} - router.PathPrefix("/").Handler(lattice) + // Endpoints to support lattice UI embedded via statik. + // The messiness here reflects the fact that assets live in a nontrivial + // directory structure that is controlled externally. + latticeHandler := NewStatikHandler(handler) + router.PathPrefix("/static").Handler(latticeHandler) + router.Path("/").Handler(latticeHandler) + router.Path("/vds").Handler(latticeHandler) + router.Path("/favicon.png").Handler(latticeHandler) + router.Path("/favicon.svg").Handler(latticeHandler) + router.Path("/manifest.json").Handler(latticeHandler) router.Use(handler.queryArgValidator) router.Use(handler.addQueryContext) @@ -469,20 +442,56 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.Handler.ServeHTTP(w, r) } -func (h *Handler) handleLattice(w http.ResponseWriter, r *http.Request) { - // If user is using curl, don't chuck HTML at them +// statikHandler implements the http.Handler interface, and responds to +// requests for static assets with the appropriate file contents embedded +// in a statik filesystem. +type statikHandler struct { + handler *Handler + statikFS http.FileSystem +} + +// NewStatikHandler returns a new instance of statikHandler +func NewStatikHandler(h *Handler) statikHandler { + fs, err := h.FileSystem.New() + if err == nil { + h.logger.Printf("enabled lattice UI at %s", h.api.Node().URI) + } + + return statikHandler{ + handler: h, + statikFS: fs, + } +} + +func (s statikHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.UserAgent(), "curl") { http.Error(w, "Welcome. Pilosa is running. Visit https://www.pilosa.com/docs/ for more information or try the Lattice UI by visiting this URL in your browser.", http.StatusNotFound) return } - filesystem, err := h.FileSystem.New() - if err != nil { - _ = h.writeQueryResponse(w, r, &pilosa.QueryResponse{Err: err}) - h.logger.Printf("Lattice UI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") + // /vds is a front-end route, not a backend route. Without this check, refreshing at /vds + // will request a nonexistent resource and return 404. + if r.URL.String() == "/vds" { + url, _ := url.Parse("/") + r.URL = url + } + + if s.statikFS == nil { + msg := "Lattice UI is not available. Please run `make generate-statik` before building Pilosa with `make install`." + s.handler.logger.Printf(msg) + http.Error(w, msg, http.StatusInternalServerError) return } - http.FileServer(filesystem).ServeHTTP(w, r) + http.FileServer(s.statikFS).ServeHTTP(w, r) + /* + filesystem, err := s.handler.FileSystem.New() // TODO + if err != nil { + s.handler.logger.Printf("Lattice UI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + http.FileServer(filesystem).ServeHTTP(w, r) + */ } // successResponse is a general success/error struct for http responses. From 99869ce79256c14374300eaf81f222d6695f6e62 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 27 Aug 2020 18:12:03 -0500 Subject: [PATCH 05/11] Minor fixes --- Makefile | 6 ++++-- README.md | 2 ++ http/handler.go | 12 ++---------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 6b7ccfaf5..76b6d5487 100644 --- a/Makefile +++ b/Makefile @@ -135,9 +135,11 @@ prerelease-upload: install: go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa -build-lattice: require-yarn +lattice: git clone git@github.com:molecula/lattice.git - cd lattice && yarn install && yarn build + +build-lattice: lattice require-yarn + cd lattice && git pull && yarn install && yarn build # `go generate` protocol buffers generate-protoc: require-protoc require-protoc-gen-gofast diff --git a/README.md b/README.md index 8057a469d..4543a3524 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ See our [Documentation](https://www.pilosa.com/docs/) for information about inst 1. [Install Pilosa](https://www.pilosa.com/docs/installation/). +Optionally, to include Lattice, the in-browser UI, follow the "Build from source" instructions, and run `make generate-statik` before `make install`. When you run a local Pilosa server on the default host, for example, you can access Lattice at [localhost:10101](http://localhost:10101). + 2. [Start Pilosa](https://www.pilosa.com/docs/getting-started/#starting-pilosa) with the default configuration: ```shell diff --git a/http/handler.go b/http/handler.go index b1981c901..24e7e8142 100644 --- a/http/handler.go +++ b/http/handler.go @@ -406,6 +406,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/index/{index}/field/{field}/remote-available-shards/{shardID}", handler.handleDeleteRemoteAvailableShard).Methods("DELETE") router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.handlePostTranslateIndexDB).Methods("POST").Name("PostTranslateIndexDB") router.HandleFunc("/internal/translate/field/{index}/{field}", handler.handlePostTranslateFieldDB).Methods("POST").Name("PostTranslateFieldDB") @@ -454,7 +455,7 @@ type statikHandler struct { func NewStatikHandler(h *Handler) statikHandler { fs, err := h.FileSystem.New() if err == nil { - h.logger.Printf("enabled lattice UI at %s", h.api.Node().URI) + h.logger.Printf("enabled Lattice UI at %s", h.api.Node().URI) } return statikHandler{ @@ -483,15 +484,6 @@ func (s statikHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } http.FileServer(s.statikFS).ServeHTTP(w, r) - /* - filesystem, err := s.handler.FileSystem.New() // TODO - if err != nil { - s.handler.logger.Printf("Lattice UI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - http.FileServer(filesystem).ServeHTTP(w, r) - */ } // successResponse is a general success/error struct for http responses. From 6e917b6a9ad7564671d76cfbc9823f9043269fe9 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 27 Aug 2020 19:26:14 -0500 Subject: [PATCH 06/11] Log lattice version info --- Makefile | 3 ++- api.go | 5 +++++ http/handler.go | 2 +- version.go | 5 +++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 76b6d5487..246e59f0c 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) +LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD || echo none) VARIANT = Molecula VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH) BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) @@ -9,7 +10,7 @@ BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) BUILD_TIME := $(shell date -u +%FT%T%z) SHARD_WIDTH = 20 COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) -LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT)" +LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)" GO_VERSION=latest RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) diff --git a/api.go b/api.go index 05808f61d..773c73ae4 100644 --- a/api.go +++ b/api.go @@ -1626,6 +1626,11 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } +// Version returns the Lattice version. +func (api *API) LatticeVersion() string { + return LatticeVersionInfo() +} + // Info returns information about this server instance. func (api *API) Info() serverInfo { si := api.server.systemInfo diff --git a/http/handler.go b/http/handler.go index 24e7e8142..f522baf32 100644 --- a/http/handler.go +++ b/http/handler.go @@ -455,7 +455,7 @@ type statikHandler struct { func NewStatikHandler(h *Handler) statikHandler { fs, err := h.FileSystem.New() if err == nil { - h.logger.Printf("enabled Lattice UI at %s", h.api.Node().URI) + h.logger.Printf("enabled Lattice UI (%s) at %s", h.api.LatticeVersion(), h.api.Node().URI) } return statikHandler{ diff --git a/version.go b/version.go index 8671d683f..4313c9e33 100644 --- a/version.go +++ b/version.go @@ -20,6 +20,7 @@ var Version string var Commit string var Variant string var BuildTime string +var LatticeCommit string func VersionInfo() string { var prefix string @@ -50,3 +51,7 @@ func VersionInfo() string { return prefix + "Pilosa" + suffix } + +func LatticeVersionInfo() string { + return "g" + LatticeCommit +} From 659a2bb560e5c66371ee06039a61fc95873ab4af Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 27 Aug 2020 20:12:51 -0500 Subject: [PATCH 07/11] Silence stderr in makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 246e59f0c..241904367 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) -LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD || echo none) +LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null) VARIANT = Molecula VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH) BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) From 384b6511cd22d494f8aa048aaafa8528ca3bae51 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 27 Aug 2020 22:38:49 -0500 Subject: [PATCH 08/11] Replace null with [] in /schema field response --- holder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/holder.go b/holder.go index abe21b9ce..c531039a4 100644 --- a/holder.go +++ b/holder.go @@ -756,6 +756,7 @@ func (h *Holder) limitedSchema() []*IndexInfo { CreatedAt: index.CreatedAt(), Options: index.Options(), ShardWidth: ShardWidth, + Fields: make([]*FieldInfo, 0, len(index.Fields())), } for _, field := range index.Fields() { if strings.HasPrefix(field.name, "_") { From f9d30408275e67b76797258c9d06055c6ac335e8 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 28 Aug 2020 00:33:54 -0500 Subject: [PATCH 09/11] Update gitignore and makefile --- .gitignore | 1 + Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8468c5367..8c58b64ee 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ vendor .DS_Store build *~ +lattice \ No newline at end of file diff --git a/Makefile b/Makefile index 241904367..93e5407f3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build require-statik test testv testv-race testvsub testvsub-race +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) From a3e122dc5ef16dfdb27404e65cf9535278c5cfca Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 28 Aug 2020 19:58:25 -0500 Subject: [PATCH 10/11] Fix CORS support by applying middleware to router. --- http/handler.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/http/handler.go b/http/handler.go index f522baf32..2da8c4c67 100644 --- a/http/handler.go +++ b/http/handler.go @@ -70,6 +70,8 @@ type Handler struct { closeTimeout time.Duration server *http.Server + + middleware []func(http.Handler) http.Handler } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -94,10 +96,10 @@ type handlerOption func(s *Handler) error func OptHandlerAllowedOrigins(origins []string) handlerOption { return func(h *Handler) error { - h.Handler = handlers.CORS( + h.middleware = append(h.middleware, handlers.CORS( handlers.AllowedOrigins(origins), handlers.AllowedHeaders([]string{"Content-Type"}), - )(h.Handler) + )) return nil } } @@ -347,7 +349,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { } // newRouter creates a new mux http router. -func newRouter(handler *Handler) *mux.Router { +func newRouter(handler *Handler) http.Handler { router := mux.NewRouter() router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST").Name("PostClusterResizeAbort") router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST").Name("PostClusterResizeRemoveNode") @@ -425,7 +427,18 @@ func newRouter(handler *Handler) *mux.Router { router.Use(handler.addQueryContext) router.Use(handler.extractTracing) router.Use(handler.collectStats) - return router + var h http.Handler = router + for _, middleware := range handler.middleware { + // Ideally, we would use `router.Use` to inject middleware, + // instead of wrapping the handler. The reason we can't is + // because the router will only apply middleware to matched + // handlers. In this case, it won't match handlers with the + // OPTIONS method, needed by the CORS middleware. This issue + // is described in detail here: + // https://github.com/gorilla/handlers/issues/142 + h = middleware(h) + } + return h } // ServeHTTP handles an HTTP request. From b14ebcadae1dc0148f1334f37c1c33db17cc06c0 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Sun, 30 Aug 2020 08:16:19 -0500 Subject: [PATCH 11/11] Unexport statik filesystem --- .gitignore | 2 +- http/handler.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 8c58b64ee..7a41479b9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ vendor .DS_Store build *~ -lattice \ No newline at end of file +lattice diff --git a/http/handler.go b/http/handler.go index 2da8c4c67..c8953984c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -56,7 +56,7 @@ import ( type Handler struct { Handler http.Handler - FileSystem pilosa.FileSystem + fileSystem pilosa.FileSystem logger logger.Logger @@ -113,7 +113,7 @@ func OptHandlerAPI(api *pilosa.API) handlerOption { func OptHandlerFileSystem(fs pilosa.FileSystem) handlerOption { return func(h *Handler) error { - h.FileSystem = fs + h.fileSystem = fs return nil } } @@ -154,7 +154,7 @@ func NewHandler(opts ...handlerOption) (*Handler, error) { } }) handler := &Handler{ - FileSystem: pilosa.NopFileSystem, + fileSystem: pilosa.NopFileSystem, logger: logger.NopLogger, closeTimeout: time.Second * 30, } @@ -466,7 +466,7 @@ type statikHandler struct { // NewStatikHandler returns a new instance of statikHandler func NewStatikHandler(h *Handler) statikHandler { - fs, err := h.FileSystem.New() + fs, err := h.fileSystem.New() if err == nil { h.logger.Printf("enabled Lattice UI (%s) at %s", h.api.LatticeVersion(), h.api.Node().URI) }