From 0a8d573fb396b05bfe06db217b08ae5552c1a6e7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 16 Apr 2018 17:12:28 -0500 Subject: [PATCH 1/3] WIP: Remove SecurityManager. Implement api restrictions in api package. --- api.go | 246 ++++++++++++++++++++++++++++++++++++++++- cluster.go | 12 +- handler.go | 59 +++------- handler_test.go | 2 +- security_manager.go | 32 ------ server.go | 4 +- server/cluster_test.go | 3 + test/cluster.go | 1 + test/handler.go | 2 - 9 files changed, 262 insertions(+), 99 deletions(-) delete mode 100644 security_manager.go diff --git a/api.go b/api.go index c447ef9f2..970766842 100644 --- a/api.go +++ b/api.go @@ -59,8 +59,30 @@ func NewAPI() *API { } } +// functionStates specifies the api functions that are valid for each +// cluster state. +var functionStates = map[string][]int{ + ClusterStateStarting: functionCommon, + ClusterStateNormal: append(functionCommon, functionNormal...), + ClusterStateResizing: append(functionCommon, functionResizing...), +} + +func (api *API) validate(f int) error { + state := api.Cluster.State() + for _, fnc := range functionStates[state] { + if f == fnc { + return nil + } + } + return fmt.Errorf("api function not allowed in state %s", state) +} + // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { + if err := api.validate(apiQuery); err != nil { + return QueryResponse{}, errors.Wrap(err, "validate api function: query") + } + resp := QueryResponse{} q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() @@ -125,6 +147,10 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { + if err := api.validate(apiCreateIndex); err != nil { + return nil, errors.Wrap(err, "validate api function: create index") + } + // Create index. index, err := api.Holder.CreateIndex(indexName, options) if err != nil { @@ -146,6 +172,10 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index // Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { + if err := api.validate(apiIndex); err != nil { + return nil, errors.Wrap(err, "validate api function: index") + } + index := api.Holder.Index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -156,6 +186,10 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { // DeleteIndex removes the named index. If the index is not found it does // nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { + if err := api.validate(apiDeleteIndex); err != nil { + return errors.Wrap(err, "validate api function: delete index") + } + // Delete index from the holder. err := api.Holder.DeleteIndex(indexName) if err != nil { @@ -176,6 +210,10 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { + if err := api.validate(apiCreateFrame); err != nil { + return nil, errors.Wrap(err, "validate api function: create frame") + } + // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -207,6 +245,10 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // found, an error is returned. If the frame is not found, it is ignored and no // action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { + if err := api.validate(apiDeleteFrame); err != nil { + return errors.Wrap(err, "validate api function: delete frame") + } + // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -235,6 +277,10 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // ExportCSV encodes the fragment designated by the index,frame,view,slice as // CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { + if err := api.validate(apiExportCSV); err != nil { + return errors.Wrap(err, "validate api function: export csv") + } + // Validate that this handler owns the slice. if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) @@ -267,14 +313,22 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin } // SliceNodes returns the node and all replicas which should contain a slice's data. -func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) []*Node { - return api.Cluster.SliceNodes(indexName, slice) +func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { + if err := api.validate(apiSliceNodes); err != nil { + return nil, errors.Wrap(err, "validate api function: slice nodes") + } + + return api.Cluster.SliceNodes(indexName, slice), nil } // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { + if err := api.validate(apiMarshalFragment); err != nil { + return nil, errors.Wrap(err, "validate api function: marshal fragment") + } + // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -287,6 +341,10 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // Reader which was previously written by MarshalFragment to populate the // fragment's data. func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { + if err := api.validate(apiUnmarshalFragment); err != nil { + return errors.Wrap(err, "validate api function: unmarshal fragment") + } + // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -316,6 +374,10 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // return anything useful. Currently it returns protobuf encoded row and column // ids from a "block" which is a subdivision of a fragment. func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { + if err := api.validate(apiFragmentBlockData); err != nil { + return nil, errors.Wrap(err, "validate api function: fragment block data") + } + reqBytes, err := ioutil.ReadAll(body) if err != nil { return nil, BadRequestError{errors.Wrap(err, "read body error")} @@ -337,7 +399,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // Encode response. buf, err := proto.Marshal(&resp) if err != nil { - return nil, errors.Wrap(err, "merge block response encoding error: %s") + return nil, errors.Wrap(err, "merge block response encoding error") } return buf, nil @@ -345,6 +407,10 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { + if err := api.validate(apiFragmentBlocks); err != nil { + return nil, errors.Wrap(err, "validate api function: fragment blocks") + } + // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -359,6 +425,10 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // RestoreFrame reads all the data that this host should have for a given frame // from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { + if err := api.validate(apiRestoreFrame); err != nil { + return errors.Wrap(err, "validate api function: restore frame") + } + // Create a client for the remote cluster. client := NewInternalHTTPClientFromURI(host, api.RemoteClient) @@ -433,6 +503,10 @@ func (api *API) Hosts(ctx context.Context) []*Node { // CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { + if err := api.validate(apiCreateInputDefinition); err != nil { + return errors.Wrap(err, "validate api function: create input definition") + } + api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. Please open an issue if you need to continue using it.`) // Find index. @@ -467,6 +541,10 @@ Please open an issue if you need to continue using it.`) // InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { + if err := api.validate(apiInputDefinition); err != nil { + return nil, errors.Wrap(err, "validate api function: input definition") + } + api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) // Find index. index := api.Holder.Index(indexName) @@ -483,6 +561,10 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN // DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { + if err := api.validate(apiDeleteInputDefinition); err != nil { + return errors.Wrap(err, "validate api function: delete input definition") + } + api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) @@ -508,6 +590,10 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp // WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { + if err := api.validate(apiWriteInput); err != nil { + return errors.Wrap(err, "validate api function: write input") + } + api.Logger.Printf("WriteInput is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) @@ -532,6 +618,10 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { + if err := api.validate(apiRecalculateCaches); err != nil { + return errors.Wrap(err, "validate api function: recalculate caches") + } + err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) if err != nil { return errors.Wrap(err, "broacasting message") @@ -542,7 +632,11 @@ func (api *API) RecalculateCaches(ctx context.Context) error { // PostClusterMessage is for internal use. It decodes a protobuf message out of // the body and forwards it to the BroadcastHandler. -func (api *API) PostClusterMessage(ctx context.Context, reqBody io.Reader) error { +func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { + if err := api.validate(apiClusterMessage); err != nil { + return errors.Wrap(err, "validate api function: cluster message") + } + // Read entire body. body, err := ioutil.ReadAll(reqBody) if err != nil { @@ -575,6 +669,10 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { + if err := api.validate(apiCreateField); err != nil { + return errors.Wrap(err, "validate api function: create field") + } + // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -601,6 +699,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { + if err := api.validate(apiDeleteField); err != nil { + return errors.Wrap(err, "validate api function: delete field") + } + // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -627,6 +729,10 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str // Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { + if err := api.validate(apiFields); err != nil { + return nil, errors.Wrap(err, "validate api function: fields") + } + index := api.Holder.index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -642,6 +748,10 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { + if err := api.validate(apiViews); err != nil { + return nil, errors.Wrap(err, "validate api function: views") + } + // Retrieve views. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -655,6 +765,10 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { + if err := api.validate(apiDeleteView); err != nil { + return errors.Wrap(err, "validate api function: delete view") + } + // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -685,6 +799,10 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiIndexAttrDiff); err != nil { + return nil, errors.Wrap(err, "validate api function: index attr diff") + } + // Retrieve index from holder. index := api.Holder.Index(indexName) if index == nil { @@ -715,6 +833,10 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At } func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiFrameAttrDiff); err != nil { + return nil, errors.Wrap(err, "validate api function: frame attr diff") + } + // Retrieve index from holder. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -746,6 +868,10 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { + if err := api.validate(apiImport); err != nil { + return errors.Wrap(err, "validate api function: import") + } + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { return err @@ -771,6 +897,10 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { + if err := api.validate(apiImportValue); err != nil { + return errors.Wrap(err, "validate api function: import value") + } + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { return err @@ -786,6 +916,10 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { + if err := api.validate(apiModifyIndexTimeQuantum); err != nil { + return errors.Wrap(err, "validate api function: modify index time quantum") + } + // Retrieve index by name. index := api.Holder.Index(indexName) if index == nil { @@ -799,6 +933,10 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti // ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO: // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { + if err := api.validate(apiModifyFrameTimeQuantum); err != nil { + return errors.Wrap(err, "validate api function: modify frame time quantum") + } + // Retrieve index by name. frame := api.Holder.Frame(indexName, frameName) if frame == nil { @@ -933,6 +1071,10 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na // SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { + if err := api.validate(apiSetCoordinator); err != nil { + return nil, nil, errors.Wrap(err, "validate api function: set coordinator") + } + oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) newNode = api.Cluster.nodeByID(id) if newNode == nil { @@ -959,6 +1101,10 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // RemoveNode puts the cluster into the "RESIZING" state and begins the job of // removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { + if err := api.validate(apiRemoveNode); err != nil { + return nil, errors.Wrap(err, "validate api function: remove node") + } + removeNode := api.Cluster.nodeByID(id) if removeNode == nil { return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") @@ -974,6 +1120,10 @@ func (api *API) RemoveNode(id string) (*Node, error) { // ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { + if err := api.validate(apiResizeAbort); err != nil { + return errors.Wrap(err, "validate api function: resize abort") + } + if !api.Cluster.IsCoordinator() { return ErrNodeNotCoordinator } @@ -992,3 +1142,91 @@ func (api *API) State() string { func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } + +// API validation constants. +const ( + apiClusterMessage int = iota + apiCreateField + apiCreateFrame + apiCreateIndex + apiCreateInputDefinition + apiDeleteField + apiDeleteFrame + apiDeleteIndex + apiDeleteInputDefinition + apiDeleteView + apiExportCSV + apiFields + apiFragmentBlockData + apiFragmentBlocks + apiFrameAttrDiff + //apiHosts // not implemented + apiImport + apiImportValue + apiIndex + apiIndexAttrDiff + apiInputDefinition + //apiLocalID // not implemented + //apiLongQueryTime // not implemented + apiMarshalFragment + //apiMaxInverseSlices // not implemented + //apiMaxSlices // not implemented + apiModifyFrameTimeQuantum + apiModifyIndexTimeQuantum + apiQuery + apiRecalculateCaches + apiRemoveNode + apiResizeAbort + apiRestoreFrame + //apiSchema // not implemented + apiSetCoordinator + apiSliceNodes + //apiState // not implemented + //apiStatsWithTags // not implemented + apiUnmarshalFragment + //apiVersion // not implemented + apiViews + apiWriteInput +) + +var functionCommon = []int{ + apiClusterMessage, + apiMarshalFragment, + apiSetCoordinator, +} + +var functionResizing = []int{ + apiResizeAbort, +} + +var functionNormal = []int{ + apiCreateField, + apiCreateFrame, + apiCreateIndex, + apiCreateInputDefinition, + apiDeleteField, + apiDeleteFrame, + apiDeleteIndex, + apiDeleteInputDefinition, + apiDeleteView, + apiExportCSV, + apiFields, + apiFragmentBlockData, + apiFragmentBlocks, + apiFrameAttrDiff, + apiImport, + apiImportValue, + apiIndex, + apiIndexAttrDiff, + apiInputDefinition, + apiModifyFrameTimeQuantum, + apiModifyIndexTimeQuantum, + apiQuery, + apiRecalculateCaches, + apiRemoveNode, + apiRestoreFrame, + apiSliceNodes, + apiUnmarshalFragment, + apiViews, + apiWriteInput, +} diff --git a/cluster.go b/cluster.go index c7945d938..7e871698d 100644 --- a/cluster.go +++ b/cluster.go @@ -264,7 +264,6 @@ type Cluster struct { // Close management wg sync.WaitGroup closing chan struct{} - prefect SecurityManager Logger Logger @@ -285,8 +284,7 @@ func NewCluster() *Cluster { closing: make(chan struct{}), joining: make(chan struct{}), - Logger: NopLogger, - prefect: &NopSecurityManager{}, + Logger: NopLogger, } } @@ -433,19 +431,11 @@ func (c *Cluster) setState(state string) { var doCleanup bool switch state { - case ClusterStateResizing: - c.prefect.SetRestricted() case ClusterStateNormal: - c.prefect.SetNormal() - // Don't change routing for these states: - // - ClusterStateStarting - // If state is RESIZING -> NORMAL then run cleanup. if c.state == ClusterStateResizing { doCleanup = true } - default: - panic(fmt.Sprintf("invalid cluster state: %s", state)) } c.state = state diff --git a/handler.go b/handler.go index bc6c625ef..4e20fcb07 100644 --- a/handler.go +++ b/handler.go @@ -43,9 +43,7 @@ import ( type Handler struct { Router *mux.Router - FileSystem FileSystem - NormalRouter *mux.Router - RestrictedRouter *mux.Router + FileSystem FileSystem // The execution engine for running queries. Executor interface { @@ -83,29 +81,11 @@ func NewHandler() *Handler { FileSystem: NopFileSystem, Logger: NopLogger, } - BuildRouters(handler) + handler.Router = NewRouter(handler) handler.populateValidators() return handler } -// BuildRouters creates Gorilla Mux http routers for both normal and restricted endpoints. -func BuildRouters(handler *Handler) { - router := mux.NewRouter() - loadCommon(router, handler) - loadNormal(router, handler) - handler.NormalRouter = router - router.Use(handler.queryArgValidator) - - // Restricted router. - router = mux.NewRouter() - loadCommon(router, handler) - loadRestricted(router, handler) - handler.RestrictedRouter = router - router.Use(handler.queryArgValidator) - - handler.SetRestricted() -} - func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") @@ -138,17 +118,9 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { }) } -// SetNormal is a method of the SecurityManager interface which provides normal URI routing. -func (h *Handler) SetNormal() { - h.Router = h.NormalRouter -} - -// SetRestricted is a method of the SecurityManager interface which provides restricted URI routing. -func (h *Handler) SetRestricted() { - h.Router = h.RestrictedRouter -} - -func loadCommon(router *mux.Router, handler *Handler) { +// NewRouter creates a new mux http router. +func NewRouter(handler *Handler) *mux.Router { + router := mux.NewRouter() router.HandleFunc("/", handler.handleWebUI).Methods("GET") router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") @@ -162,16 +134,9 @@ func loadCommon(router *mux.Router, handler *Handler) { router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") - router.Use(handler.queryArgValidator) -} -func loadRestricted(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - router.NotFoundHandler = http.HandlerFunc(handler.reportRestricted) - router.Use(handler.queryArgValidator) -} -func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.Handle("/debug/vars", expvar.Handler()).Methods("GET") @@ -212,10 +177,8 @@ func loadNormal(router *mux.Router, handler *Handler) { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") -} - -func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) { - http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.API.State()), http.StatusMethodNotAllowed) + router.Use(handler.queryArgValidator) + return router } func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { @@ -1169,7 +1132,11 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } // Retrieve fragment owner nodes. - nodes := h.API.SliceNodes(r.Context(), index, slice) + nodes, err := h.API.SliceNodes(r.Context(), index, slice) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { @@ -1727,7 +1694,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques return } - err := h.API.PostClusterMessage(r.Context(), r.Body) + err := h.API.ClusterMessage(r.Context(), r.Body) if err != nil { // TODO this was the previous behavior, but perhaps not everything is a bad request http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/handler_test.go b/handler_test.go index 063421667..78f9e470f 100644 --- a/handler_test.go +++ b/handler_test.go @@ -160,7 +160,7 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { t.Run("No resize job", func(t *testing.T) { h := test.NewHandler() h.API.Cluster = test.NewCluster(1) - h.SetRestricted() + h.API.Cluster.SetState(pilosa.ClusterStateResizing) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) diff --git a/security_manager.go b/security_manager.go deleted file mode 100644 index 80b696c38..000000000 --- a/security_manager.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -// SecurityManager provides the ability to limit access to restricted endpoints -// during cluster configuration. -type SecurityManager interface { - SetRestricted() - SetNormal() -} - -// NopSecurityManager provides a no-op implementation of the SecurityManager interface. -type NopSecurityManager struct { -} - -// SetRestricted no-op. -func (sdm *NopSecurityManager) SetRestricted() {} - -// SetNormal no-op. -func (sdm *NopSecurityManager) SetNormal() {} diff --git a/server.go b/server.go index bb629b66a..049672686 100644 --- a/server.go +++ b/server.go @@ -169,10 +169,8 @@ func (s *Server) Open() error { s.Handler.API.StatusHandler = s s.Handler.API.URI = s.URI s.Handler.API.Cluster = s.Cluster - s.Handler.Executor = e - - s.Cluster.prefect = s.Handler s.Handler.API.Executor = e + s.Handler.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster diff --git a/server/cluster_test.go b/server/cluster_test.go index 04d53b438..7f4886612 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -101,6 +101,9 @@ func TestMain_SendReceiveMessage(t *testing.T) { t.Fatal(err) } + m0.Server.Cluster.SetState(pilosa.ClusterStateNormal) + m1.Server.Cluster.SetState(pilosa.ClusterStateNormal) + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Expected indexes and Frames diff --git a/test/cluster.go b/test/cluster.go index 308a7e69c..77cf496a8 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -50,6 +50,7 @@ func NewCluster(n int) *pilosa.Cluster { c.Node = c.Nodes[0] c.Coordinator = c.Nodes[0].ID + c.SetState(pilosa.ClusterStateNormal) return c } diff --git a/test/handler.go b/test/handler.go index cb90cd383..c325d3243 100644 --- a/test/handler.go +++ b/test/handler.go @@ -47,8 +47,6 @@ func NewHandler() *Handler { // Handler test messages can no-op. h.API.Broadcaster = pilosa.NopBroadcaster - h.SetNormal() - return h } From 54680652137aee9b16cb4999b15d66d6c27020fa Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Apr 2018 13:01:36 -0500 Subject: [PATCH 2/3] improve apiFunc error handling. change slice to map in function validation. --- Makefile | 6 +- api.go | 173 +++++++++++++++++++++++++--------------------- apifunc_string.go | 16 +++++ pilosa.go | 6 ++ 4 files changed, 120 insertions(+), 81 deletions(-) create mode 100644 apifunc_string.go diff --git a/Makefile b/Makefile index 9e2c6fa3f..cab932fba 100644 --- a/Makefile +++ b/Makefile @@ -80,8 +80,12 @@ generate-protoc: require-protoc require-protoc-gen-gofast generate-statik: require-statik go generate github.com/pilosa/pilosa/statik +# `go generate` stringers +generate-stringer: + go generate github.com/pilosa/pilosa + # `go generate` all needed packages -generate: generate-protoc generate-statik +generate: generate-protoc generate-statik generate-stringer # Create Docker image from Dockerfile docker: diff --git a/api.go b/api.go index 970766842..6852a1f87 100644 --- a/api.go +++ b/api.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:generate stringer -type=apiFunc + package pilosa import ( @@ -61,26 +63,35 @@ func NewAPI() *API { // functionStates specifies the api functions that are valid for each // cluster state. -var functionStates = map[string][]int{ +var functionStates = map[string]map[apiFunc]struct{}{ ClusterStateStarting: functionCommon, - ClusterStateNormal: append(functionCommon, functionNormal...), - ClusterStateResizing: append(functionCommon, functionResizing...), + ClusterStateNormal: appendMap(functionCommon, functionNormal), + ClusterStateResizing: appendMap(functionCommon, functionResizing), } -func (api *API) validate(f int) error { - state := api.Cluster.State() - for _, fnc := range functionStates[state] { - if f == fnc { - return nil - } +func appendMap(a, b map[apiFunc]struct{}) map[apiFunc]struct{} { + r := make(map[apiFunc]struct{}) + for k, v := range a { + r[k] = v } - return fmt.Errorf("api function not allowed in state %s", state) + for k, v := range b { + r[k] = v + } + return r +} + +func (api *API) validate(f apiFunc) error { + state := api.Cluster.State() + if _, ok := functionStates[state][f]; ok { + return nil + } + return ApiFunctionNotAllowedError{errors.Errorf("api function %s not allowed in state %s", f, state)} } // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { if err := api.validate(apiQuery); err != nil { - return QueryResponse{}, errors.Wrap(err, "validate api function: query") + return QueryResponse{}, errors.Wrap(err, "validate api function") } resp := QueryResponse{} @@ -148,7 +159,7 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { if err := api.validate(apiCreateIndex); err != nil { - return nil, errors.Wrap(err, "validate api function: create index") + return nil, errors.Wrap(err, "validate api function") } // Create index. @@ -173,7 +184,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index // Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { if err := api.validate(apiIndex); err != nil { - return nil, errors.Wrap(err, "validate api function: index") + return nil, errors.Wrap(err, "validate api function") } index := api.Holder.Index(indexName) @@ -187,7 +198,7 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { // nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { if err := api.validate(apiDeleteIndex); err != nil { - return errors.Wrap(err, "validate api function: delete index") + return errors.Wrap(err, "validate api function") } // Delete index from the holder. @@ -211,7 +222,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { if err := api.validate(apiCreateFrame); err != nil { - return nil, errors.Wrap(err, "validate api function: create frame") + return nil, errors.Wrap(err, "validate api function") } // Find index. @@ -246,7 +257,7 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { if err := api.validate(apiDeleteFrame); err != nil { - return errors.Wrap(err, "validate api function: delete frame") + return errors.Wrap(err, "validate api function") } // Find index. @@ -278,7 +289,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { - return errors.Wrap(err, "validate api function: export csv") + return errors.Wrap(err, "validate api function") } // Validate that this handler owns the slice. @@ -315,7 +326,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin // SliceNodes returns the node and all replicas which should contain a slice's data. func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { if err := api.validate(apiSliceNodes); err != nil { - return nil, errors.Wrap(err, "validate api function: slice nodes") + return nil, errors.Wrap(err, "validate api function") } return api.Cluster.SliceNodes(indexName, slice), nil @@ -326,7 +337,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) // the UnmarshalFragment API call. func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { - return nil, errors.Wrap(err, "validate api function: marshal fragment") + return nil, errors.Wrap(err, "validate api function") } // Retrieve fragment from holder. @@ -342,7 +353,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // fragment's data. func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { - return errors.Wrap(err, "validate api function: unmarshal fragment") + return errors.Wrap(err, "validate api function") } // Retrieve frame. @@ -375,7 +386,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // ids from a "block" which is a subdivision of a fragment. func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { if err := api.validate(apiFragmentBlockData); err != nil { - return nil, errors.Wrap(err, "validate api function: fragment block data") + return nil, errors.Wrap(err, "validate api function") } reqBytes, err := ioutil.ReadAll(body) @@ -408,7 +419,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { - return nil, errors.Wrap(err, "validate api function: fragment blocks") + return nil, errors.Wrap(err, "validate api function") } // Retrieve fragment from holder. @@ -426,7 +437,7 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { if err := api.validate(apiRestoreFrame); err != nil { - return errors.Wrap(err, "validate api function: restore frame") + return errors.Wrap(err, "validate api function") } // Create a client for the remote cluster. @@ -504,7 +515,7 @@ func (api *API) Hosts(ctx context.Context) []*Node { // CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { if err := api.validate(apiCreateInputDefinition); err != nil { - return errors.Wrap(err, "validate api function: create input definition") + return errors.Wrap(err, "validate api function") } api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. @@ -542,7 +553,7 @@ Please open an issue if you need to continue using it.`) // InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { if err := api.validate(apiInputDefinition); err != nil { - return nil, errors.Wrap(err, "validate api function: input definition") + return nil, errors.Wrap(err, "validate api function") } api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) @@ -562,7 +573,7 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN // DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { if err := api.validate(apiDeleteInputDefinition); err != nil { - return errors.Wrap(err, "validate api function: delete input definition") + return errors.Wrap(err, "validate api function") } api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") @@ -591,7 +602,7 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp // WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { if err := api.validate(apiWriteInput); err != nil { - return errors.Wrap(err, "validate api function: write input") + return errors.Wrap(err, "validate api function") } api.Logger.Printf("WriteInput is deprecated and will be removed.") @@ -619,7 +630,7 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { if err := api.validate(apiRecalculateCaches); err != nil { - return errors.Wrap(err, "validate api function: recalculate caches") + return errors.Wrap(err, "validate api function") } err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) @@ -634,7 +645,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { // the body and forwards it to the BroadcastHandler. func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { if err := api.validate(apiClusterMessage); err != nil { - return errors.Wrap(err, "validate api function: cluster message") + return errors.Wrap(err, "validate api function") } // Read entire body. @@ -670,7 +681,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { if err := api.validate(apiCreateField); err != nil { - return errors.Wrap(err, "validate api function: create field") + return errors.Wrap(err, "validate api function") } // Retrieve frame by name. @@ -700,7 +711,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { if err := api.validate(apiDeleteField); err != nil { - return errors.Wrap(err, "validate api function: delete field") + return errors.Wrap(err, "validate api function") } // Retrieve frame by name. @@ -730,7 +741,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str // Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { if err := api.validate(apiFields); err != nil { - return nil, errors.Wrap(err, "validate api function: fields") + return nil, errors.Wrap(err, "validate api function") } index := api.Holder.index(indexName) @@ -749,7 +760,7 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { - return nil, errors.Wrap(err, "validate api function: views") + return nil, errors.Wrap(err, "validate api function") } // Retrieve views. @@ -766,7 +777,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { if err := api.validate(apiDeleteView); err != nil { - return errors.Wrap(err, "validate api function: delete view") + return errors.Wrap(err, "validate api function") } // Retrieve frame. @@ -800,7 +811,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiIndexAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function: index attr diff") + return nil, errors.Wrap(err, "validate api function") } // Retrieve index from holder. @@ -834,7 +845,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiFrameAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function: frame attr diff") + return nil, errors.Wrap(err, "validate api function") } // Retrieve index from holder. @@ -869,7 +880,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { - return errors.Wrap(err, "validate api function: import") + return errors.Wrap(err, "validate api function") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -898,7 +909,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { - return errors.Wrap(err, "validate api function: import value") + return errors.Wrap(err, "validate api function") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -917,7 +928,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyIndexTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function: modify index time quantum") + return errors.Wrap(err, "validate api function") } // Retrieve index by name. @@ -934,7 +945,7 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyFrameTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function: modify frame time quantum") + return errors.Wrap(err, "validate api function") } // Retrieve index by name. @@ -1072,7 +1083,7 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na // SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { if err := api.validate(apiSetCoordinator); err != nil { - return nil, nil, errors.Wrap(err, "validate api function: set coordinator") + return nil, nil, errors.Wrap(err, "validate api function") } oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) @@ -1102,7 +1113,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { if err := api.validate(apiRemoveNode); err != nil { - return nil, errors.Wrap(err, "validate api function: remove node") + return nil, errors.Wrap(err, "validate api function") } removeNode := api.Cluster.nodeByID(id) @@ -1121,7 +1132,7 @@ func (api *API) RemoveNode(id string) (*Node, error) { // ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { if err := api.validate(apiResizeAbort); err != nil { - return errors.Wrap(err, "validate api function: resize abort") + return errors.Wrap(err, "validate api function") } if !api.Cluster.IsCoordinator() { @@ -1143,9 +1154,11 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } +type apiFunc int + // API validation constants. const ( - apiClusterMessage int = iota + apiClusterMessage apiFunc = iota apiCreateField apiCreateFrame apiCreateIndex @@ -1189,44 +1202,44 @@ const ( apiWriteInput ) -var functionCommon = []int{ - apiClusterMessage, - apiMarshalFragment, - apiSetCoordinator, +var functionCommon = map[apiFunc]struct{}{ + apiClusterMessage: struct{}{}, + apiMarshalFragment: struct{}{}, + apiSetCoordinator: struct{}{}, } -var functionResizing = []int{ - apiResizeAbort, +var functionResizing = map[apiFunc]struct{}{ + apiResizeAbort: struct{}{}, } -var functionNormal = []int{ - apiCreateField, - apiCreateFrame, - apiCreateIndex, - apiCreateInputDefinition, - apiDeleteField, - apiDeleteFrame, - apiDeleteIndex, - apiDeleteInputDefinition, - apiDeleteView, - apiExportCSV, - apiFields, - apiFragmentBlockData, - apiFragmentBlocks, - apiFrameAttrDiff, - apiImport, - apiImportValue, - apiIndex, - apiIndexAttrDiff, - apiInputDefinition, - apiModifyFrameTimeQuantum, - apiModifyIndexTimeQuantum, - apiQuery, - apiRecalculateCaches, - apiRemoveNode, - apiRestoreFrame, - apiSliceNodes, - apiUnmarshalFragment, - apiViews, - apiWriteInput, +var functionNormal = map[apiFunc]struct{}{ + apiCreateField: struct{}{}, + apiCreateFrame: struct{}{}, + apiCreateIndex: struct{}{}, + apiCreateInputDefinition: struct{}{}, + apiDeleteField: struct{}{}, + apiDeleteFrame: struct{}{}, + apiDeleteIndex: struct{}{}, + apiDeleteInputDefinition: struct{}{}, + apiDeleteView: struct{}{}, + apiExportCSV: struct{}{}, + apiFields: struct{}{}, + apiFragmentBlockData: struct{}{}, + apiFragmentBlocks: struct{}{}, + apiFrameAttrDiff: struct{}{}, + apiImport: struct{}{}, + apiImportValue: struct{}{}, + apiIndex: struct{}{}, + apiIndexAttrDiff: struct{}{}, + apiInputDefinition: struct{}{}, + apiModifyFrameTimeQuantum: struct{}{}, + apiModifyIndexTimeQuantum: struct{}{}, + apiQuery: struct{}{}, + apiRecalculateCaches: struct{}{}, + apiRemoveNode: struct{}{}, + apiRestoreFrame: struct{}{}, + apiSliceNodes: struct{}{}, + apiUnmarshalFragment: struct{}{}, + apiViews: struct{}{}, + apiWriteInput: struct{}{}, } diff --git a/apifunc_string.go b/apifunc_string.go new file mode 100644 index 000000000..8fb9b16e0 --- /dev/null +++ b/apifunc_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type=apiFunc"; DO NOT EDIT. + +package pilosa + +import "fmt" + +const _apiFunc_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" + +var _apiFunc_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} + +func (i apiFunc) String() string { + if i < 0 || i >= apiFunc(len(_apiFunc_index)-1) { + return fmt.Sprintf("apiFunc(%d)", i) + } + return _apiFunc_name[_apiFunc_index[i]:_apiFunc_index[i+1]] +} diff --git a/pilosa.go b/pilosa.go index ffb836ae8..c3a6919db 100644 --- a/pilosa.go +++ b/pilosa.go @@ -82,6 +82,12 @@ var ( ErrResizeNotRunning = errors.New("no resize job currently running") ) +// InvalidApiFunctionError wraps an error value indicating that a particular +// API function is not allowed in the current cluster state. +type ApiFunctionNotAllowedError struct { + error +} + // BadRequestError wraps an error value to signify that a request could not be // read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest // would be returned. From e60d11a23e2d93ba5d594dac15f3d40a3d40eb0e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Apr 2018 17:07:41 -0500 Subject: [PATCH 3/3] change references from "function" to "method" --- api.go | 98 ++++++++++++++++++++++----------------------- apifunc_string.go | 16 -------- apimethod_string.go | 16 ++++++++ pilosa.go | 6 +-- 4 files changed, 68 insertions(+), 68 deletions(-) delete mode 100644 apifunc_string.go create mode 100644 apimethod_string.go diff --git a/api.go b/api.go index 6852a1f87..ed41328bf 100644 --- a/api.go +++ b/api.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate stringer -type=apiFunc +//go:generate stringer -type=apiMethod package pilosa @@ -61,16 +61,16 @@ func NewAPI() *API { } } -// functionStates specifies the api functions that are valid for each +// validAPIMethods specifies the api methods that are valid for each // cluster state. -var functionStates = map[string]map[apiFunc]struct{}{ - ClusterStateStarting: functionCommon, - ClusterStateNormal: appendMap(functionCommon, functionNormal), - ClusterStateResizing: appendMap(functionCommon, functionResizing), +var validAPIMethods = map[string]map[apiMethod]struct{}{ + ClusterStateStarting: methodsCommon, + ClusterStateNormal: appendMap(methodsCommon, methodsNormal), + ClusterStateResizing: appendMap(methodsCommon, methodsResizing), } -func appendMap(a, b map[apiFunc]struct{}) map[apiFunc]struct{} { - r := make(map[apiFunc]struct{}) +func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} { + r := make(map[apiMethod]struct{}) for k, v := range a { r[k] = v } @@ -80,18 +80,18 @@ func appendMap(a, b map[apiFunc]struct{}) map[apiFunc]struct{} { return r } -func (api *API) validate(f apiFunc) error { +func (api *API) validate(f apiMethod) error { state := api.Cluster.State() - if _, ok := functionStates[state][f]; ok { + if _, ok := validAPIMethods[state][f]; ok { return nil } - return ApiFunctionNotAllowedError{errors.Errorf("api function %s not allowed in state %s", f, state)} + return ApiMethodNotAllowedError{errors.Errorf("api method %s not allowed in state %s", f, state)} } // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { if err := api.validate(apiQuery); err != nil { - return QueryResponse{}, errors.Wrap(err, "validate api function") + return QueryResponse{}, errors.Wrap(err, "validate api method") } resp := QueryResponse{} @@ -159,7 +159,7 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { if err := api.validate(apiCreateIndex); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Create index. @@ -184,7 +184,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index // Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { if err := api.validate(apiIndex); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } index := api.Holder.Index(indexName) @@ -198,7 +198,7 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { // nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { if err := api.validate(apiDeleteIndex); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Delete index from the holder. @@ -222,7 +222,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { if err := api.validate(apiCreateFrame); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Find index. @@ -257,7 +257,7 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { if err := api.validate(apiDeleteFrame); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Find index. @@ -289,7 +289,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Validate that this handler owns the slice. @@ -326,7 +326,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin // SliceNodes returns the node and all replicas which should contain a slice's data. func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { if err := api.validate(apiSliceNodes); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } return api.Cluster.SliceNodes(indexName, slice), nil @@ -337,7 +337,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) // the UnmarshalFragment API call. func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve fragment from holder. @@ -353,7 +353,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // fragment's data. func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame. @@ -386,7 +386,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // ids from a "block" which is a subdivision of a fragment. func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { if err := api.validate(apiFragmentBlockData); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } reqBytes, err := ioutil.ReadAll(body) @@ -419,7 +419,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve fragment from holder. @@ -437,7 +437,7 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { if err := api.validate(apiRestoreFrame); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Create a client for the remote cluster. @@ -515,7 +515,7 @@ func (api *API) Hosts(ctx context.Context) []*Node { // CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { if err := api.validate(apiCreateInputDefinition); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. @@ -553,7 +553,7 @@ Please open an issue if you need to continue using it.`) // InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { if err := api.validate(apiInputDefinition); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) @@ -573,7 +573,7 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN // DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { if err := api.validate(apiDeleteInputDefinition); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") @@ -602,7 +602,7 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp // WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { if err := api.validate(apiWriteInput); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } api.Logger.Printf("WriteInput is deprecated and will be removed.") @@ -630,7 +630,7 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { if err := api.validate(apiRecalculateCaches); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) @@ -645,7 +645,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { // the body and forwards it to the BroadcastHandler. func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { if err := api.validate(apiClusterMessage); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Read entire body. @@ -681,7 +681,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { if err := api.validate(apiCreateField); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame by name. @@ -711,7 +711,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { if err := api.validate(apiDeleteField); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame by name. @@ -741,7 +741,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str // Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { if err := api.validate(apiFields); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } index := api.Holder.index(indexName) @@ -760,7 +760,7 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve views. @@ -777,7 +777,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { if err := api.validate(apiDeleteView); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame. @@ -811,7 +811,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiIndexAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve index from holder. @@ -845,7 +845,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiFrameAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve index from holder. @@ -880,7 +880,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -909,7 +909,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -928,7 +928,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyIndexTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve index by name. @@ -945,7 +945,7 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyFrameTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve index by name. @@ -1083,7 +1083,7 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na // SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { if err := api.validate(apiSetCoordinator); err != nil { - return nil, nil, errors.Wrap(err, "validate api function") + return nil, nil, errors.Wrap(err, "validate api method") } oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) @@ -1113,7 +1113,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { if err := api.validate(apiRemoveNode); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } removeNode := api.Cluster.nodeByID(id) @@ -1132,7 +1132,7 @@ func (api *API) RemoveNode(id string) (*Node, error) { // ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { if err := api.validate(apiResizeAbort); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } if !api.Cluster.IsCoordinator() { @@ -1154,11 +1154,11 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } -type apiFunc int +type apiMethod int // API validation constants. const ( - apiClusterMessage apiFunc = iota + apiClusterMessage apiMethod = iota apiCreateField apiCreateFrame apiCreateIndex @@ -1202,17 +1202,17 @@ const ( apiWriteInput ) -var functionCommon = map[apiFunc]struct{}{ +var methodsCommon = map[apiMethod]struct{}{ apiClusterMessage: struct{}{}, apiMarshalFragment: struct{}{}, apiSetCoordinator: struct{}{}, } -var functionResizing = map[apiFunc]struct{}{ +var methodsResizing = map[apiMethod]struct{}{ apiResizeAbort: struct{}{}, } -var functionNormal = map[apiFunc]struct{}{ +var methodsNormal = map[apiMethod]struct{}{ apiCreateField: struct{}{}, apiCreateFrame: struct{}{}, apiCreateIndex: struct{}{}, diff --git a/apifunc_string.go b/apifunc_string.go deleted file mode 100644 index 8fb9b16e0..000000000 --- a/apifunc_string.go +++ /dev/null @@ -1,16 +0,0 @@ -// Code generated by "stringer -type=apiFunc"; DO NOT EDIT. - -package pilosa - -import "fmt" - -const _apiFunc_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" - -var _apiFunc_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} - -func (i apiFunc) String() string { - if i < 0 || i >= apiFunc(len(_apiFunc_index)-1) { - return fmt.Sprintf("apiFunc(%d)", i) - } - return _apiFunc_name[_apiFunc_index[i]:_apiFunc_index[i+1]] -} diff --git a/apimethod_string.go b/apimethod_string.go new file mode 100644 index 000000000..8a3dce195 --- /dev/null +++ b/apimethod_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type=apiMethod"; DO NOT EDIT. + +package pilosa + +import "fmt" + +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" + +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} + +func (i apiMethod) String() string { + if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { + return fmt.Sprintf("apiMethod(%d)", i) + } + return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]] +} diff --git a/pilosa.go b/pilosa.go index c3a6919db..347fbed67 100644 --- a/pilosa.go +++ b/pilosa.go @@ -82,9 +82,9 @@ var ( ErrResizeNotRunning = errors.New("no resize job currently running") ) -// InvalidApiFunctionError wraps an error value indicating that a particular -// API function is not allowed in the current cluster state. -type ApiFunctionNotAllowedError struct { +// ApiMethodNotAllowedError wraps an error value indicating that a particular +// API method is not allowed in the current cluster state. +type ApiMethodNotAllowedError struct { error }