diff --git a/.travis.yml b/.travis.yml index 6d5674662..fbe1e0c9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,6 @@ install: - make install-dep install-statik vendor generate-statik script: - make test -# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 -after_success: - - go get github.com/mattn/goveralls - - $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go" before_deploy: - pip install awscli --user `whoami` deploy: @@ -33,6 +29,7 @@ deploy: matrix: allow_failures: - go: master + fast_finish: true notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" diff --git a/Gopkg.lock b/Gopkg.lock index f29660b22..8bb1744bd 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -82,6 +82,12 @@ revision = "1ea25387ff6f684839d82767c1733ff4d4d15d0a" version = "v1.1" +[[projects]] + name = "github.com/gorilla/handlers" + packages = ["."] + revision = "90663712d74cb411cbef281bc1e08c19d1a76145" + version = "v1.3.0" + [[projects]] name = "github.com/gorilla/mux" packages = ["."] @@ -206,22 +212,14 @@ [[projects]] name = "github.com/shirou/gopsutil" packages = [ - "cpu", "host", "internal/common", "mem", - "net", "process" ] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" -[[projects]] - branch = "master" - name = "github.com/shirou/w32" - packages = ["."] - revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" - [[projects]] branch = "master" name = "github.com/spf13/afero" @@ -304,6 +302,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "61a00007fe7398d958fc8668543d42293f1310f9f9b74c63ff96f5a892eab1e7" + inputs-digest = "8f633d73d966ca439d2fdf3704a41d8ea59be8ed9a2cab0ab73de4b72c5772ba" solver-name = "gps-cdcl" solver-version = 1 diff --git a/ctl/server.go b/ctl/server.go index 4816f6170..a4c272a46 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -33,6 +33,9 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // TLS SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) + // Handler + flags.StringSliceVarP(&srv.Config.Handler.AllowedOrigins, "handler.allowed-origins", "", []string{}, "Comma separated list of allowed origin URIs (for CORS/WebUI).") + // Cluster flags.BoolVarP(&srv.Config.Cluster.Disabled, "cluster.disabled", "", srv.Config.Cluster.Disabled, "Disabled multi-node cluster communication (used for testing)") flags.BoolVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", srv.Config.Cluster.Coordinator, "Host that will act as cluster coordinator during startup and resizing.") diff --git a/docs/configuration.md b/docs/configuration.md index c6ed8fbd4..200c91449 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -59,6 +59,18 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h bind = localhost:10101 ``` +#### CORS (Cross-Origin Resource Sharing) Allowed Origins + +* Description: List of allowed origin URIs for CORS +* Flag: `--handler.allowed-origins="https://myapp.com,https://myapp.org"` +* Env: `PILOSA_HANDLER_ALLOWED_ORIGINS="https://myapp.com,https://myapp.org"` +* Config: + + ```toml + [handler] + allowed-origins = ["https://myapp.com", "https://myapp.org"] + ``` + #### Data Dir * Description: Directory to store Pilosa data files. diff --git a/handler.go b/handler.go index 1bdb6529d..bfff395c3 100644 --- a/handler.go +++ b/handler.go @@ -31,14 +31,16 @@ import ( "time" "github.com/gogo/protobuf/proto" + "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" ) // Handler represents an HTTP handler. type Handler struct { - Router *mux.Router + Handler http.Handler FileSystem FileSystem @@ -48,6 +50,8 @@ type Handler struct { validators map[string]*queryValidationSpec API *API + + AllowedOrigins []string } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -67,15 +71,36 @@ type errorResponse struct { Error string `json:"error"` } +// HandlerOption is a functional option type for pilosa.Handler +type HandlerOption func(s *Handler) error + +func OptHandlerAllowedOrigins(origins []string) HandlerOption { + return func(h *Handler) error { + h.Handler = handlers.CORS( + handlers.AllowedOrigins(origins), + handlers.AllowedHeaders([]string{"Content-Type"}), + )(h.Handler) + return nil + } +} + // NewHandler returns a new instance of Handler with a default logger. -func NewHandler() *Handler { +func NewHandler(opts ...HandlerOption) (*Handler, error) { handler := &Handler{ FileSystem: NopFileSystem, Logger: NopLogger, } - handler.Router = NewRouter(handler) + handler.Handler = NewRouter(handler) handler.populateValidators() - return handler + + for _, opt := range opts { + err := opt(handler) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + + return handler, nil } func (h *Handler) populateValidators() { @@ -179,7 +204,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { }() t := time.Now() - h.Router.ServeHTTP(w, r) + h.Handler.ServeHTTP(w, r) dif := time.Since(t) // Calculate per request StatsD metrics when the handler is fully configured. diff --git a/handler_test.go b/handler_test.go index 24587aa0d..a7ba8fa85 100644 --- a/handler_test.go +++ b/handler_test.go @@ -37,7 +37,7 @@ import ( ) func TestHandlerPanics(t *testing.T) { - h := test.NewHandler() + h := test.MustNewHandler() bufLogger := test.NewBufferLogger() h.Handler.Logger = bufLogger @@ -65,7 +65,7 @@ func TestHandler_NotFound(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder @@ -98,7 +98,7 @@ func TestHandler_Schema(t *testing.T) { t.Fatal(err) } - h := test.NewHandler() + h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() @@ -135,7 +135,7 @@ func TestHandler_Status(t *testing.T) { t.Fatal(err) } - h := test.NewHandler() + h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) h.API.Cluster.SetState(pilosa.ClusterStateNormal) @@ -154,7 +154,7 @@ func TestHandler_Status(t *testing.T) { func TestHandler_Info(t *testing.T) { s := test.NewServer() defer s.Close() - h := test.NewHandler() + h := test.MustNewHandler() w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil)) @@ -169,7 +169,7 @@ func TestHandler_Info(t *testing.T) { func TestHandler_ClusterResizeAbort(t *testing.T) { t.Run("No resize job", func(t *testing.T) { - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Cluster.SetState(pilosa.ClusterStateResizing) @@ -198,7 +198,7 @@ func TestHandler_MaxSlices(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) - h := test.NewHandler() + h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() @@ -215,7 +215,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -243,7 +243,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -283,7 +283,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder @@ -296,7 +296,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { } func TestHandler_Query_Params_Err(t *testing.T) { w := httptest.NewRecorder() - test.NewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) + test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" { @@ -310,7 +310,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -331,7 +331,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -361,7 +361,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -394,7 +394,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { t.Fatal(err) } - h := test.NewHandler() + h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -417,7 +417,7 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -465,7 +465,7 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { t.Fatal(err) } - h := test.NewHandler() + h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -526,7 +526,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -550,7 +550,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -583,7 +583,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -604,7 +604,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { @@ -632,7 +632,7 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder w := httptest.NewRecorder() @@ -647,7 +647,7 @@ func TestHandler_Query_ErrParse(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder w := httptest.NewRecorder() @@ -704,7 +704,7 @@ func TestHandler_DeleteFrame(t *testing.T) { t.Fatal(err) } - h := test.NewHandler() + h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() @@ -1061,7 +1061,7 @@ func TestHandler_Version(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder @@ -1084,7 +1084,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(3) h.API.Cluster.ReplicaN = 2 @@ -1120,7 +1120,7 @@ func TestHandler_Expvars(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Cluster = test.NewCluster(1) h.API.Holder = hldr.Holder w := httptest.NewRecorder() @@ -1143,7 +1143,7 @@ func TestHandler_RecalculateCaches(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) @@ -1159,7 +1159,7 @@ func TestHandler_WebUI(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - h := test.NewHandler() + h := test.MustNewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) h.FileSystem = &statik.FileSystem{} @@ -1182,3 +1182,41 @@ func TestHandler_WebUI(t *testing.T) { t.Fatalf("WebUI is not being served correctly.") } } + +func TestHandler_CORS(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + + s := test.NewServer() + s.Handler.API.Holder = hldr.Holder + defer s.Close() + + // No CORS config present, so should fail + handler := test.MustNewHandler() + + req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil) + req.Header.Add("Origin", "http://test/") + req.Header.Add("Access-Control-Request-Method", "POST") + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + result := w.Result() + + // This handler does not support CORS, return Method Not Allowed (405) + if result.StatusCode != 405 { + t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode) + } + + // CORS config should allow preflight response + handler = test.MustNewHandler(pilosa.OptHandlerAllowedOrigins([]string{"http://test/"})) + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + result = w.Result() + + if result.StatusCode != 200 { + t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode) + } + if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" { + t.Fatal("CORS header not present") + } +} diff --git a/roaring/roaring.go b/roaring/roaring.go index cc8466a5f..fafce4d59 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1461,22 +1461,14 @@ func (c *Container) arrayMax() uint16 { func (c *Container) bitmapMax() uint16 { // Search bitmap in reverse order. - for i := len(c.bitmap) - 1; i >= 0; i-- { + for i := len(c.bitmap); i > 0; i-- { // If value is zero then skip. - v := c.bitmap[i] - if v == 0 { - continue + v := c.bitmap[i-1] + if v != 0 { + r := bits.LeadingZeros64(v) + return uint16((i-1)*64 + 63 - r) } - // Find the highest set bit. - for j := uint16(63); ; j-- { - if v&(1<