From 6c3f3ac0f092340338604863d19388acf9bb8a98 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 1 Mar 2018 17:39:14 +0300 Subject: [PATCH 01/79] Added API struct; moved query logic to API --- api.go | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++ handler.go | 47 ++++--------------------- server.go | 5 +++ 3 files changed, 113 insertions(+), 40 deletions(-) create mode 100644 api.go diff --git a/api.go b/api.go new file mode 100644 index 000000000..5ab4a36aa --- /dev/null +++ b/api.go @@ -0,0 +1,101 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "context" + "strings" + + "github.com/pilosa/pilosa/pql" +) + +type QueryOptions struct { + Remote bool + ExcludeAttrs bool + ExcludeBits bool + ColumnAttrs bool +} + +type API struct { + holder *Holder + // The execution engine for running queries. + executor interface { + Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) + } +} + +func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryResponse, error) { + resp := QueryResponse{} + + q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() + if err != nil { + // TODO: Wrap + return resp, err + } + execOpts := &ExecOptions{ + Remote: req.Remote, + ExcludeAttrs: req.ExcludeAttrs, + ExcludeBits: req.ExcludeBits, + } + results, err := a.executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + if err != nil { + return resp, err + } + resp.Results = results + + // Fill column attributes if requested. + if req.ColumnAttrs && !req.ExcludeBits { + // Consolidate all column ids across all calls. + var columnIDs []uint64 + for _, result := range results { + bm, ok := result.(*Bitmap) + if !ok { + continue + } + columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) + } + + // Retrieve column attributes across all calls. + columnAttrSets, err := a.readColumnAttrSets(a.holder.Index(req.Index), columnIDs) + if err != nil { + return resp, err + } + resp.ColumnAttrSets = columnAttrSets + } + return resp, nil +} + +// readColumnAttrSets returns a list of column attribute objects by id. +func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) { + if index == nil { + return nil, nil + } + + ax := make([]*ColumnAttrSet, 0, len(ids)) + for _, id := range ids { + // Read attributes for column. Skip column if empty. + attrs, err := index.ColumnAttrStore().Attrs(id) + if err != nil { + return nil, err + } else if len(attrs) == 0 { + continue + } + + // Append column with attributes. + ax = append(ax, &ColumnAttrSet{ID: id, Attrs: attrs}) + } + + return ax, nil +} diff --git a/handler.go b/handler.go index ffd28da46..c40148b43 100644 --- a/handler.go +++ b/handler.go @@ -69,6 +69,8 @@ type Handler struct { // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec + + api *API } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -328,8 +330,6 @@ type getStatusResponse struct { // handlePostQuery handles /query requests. func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - // Parse incoming request. req, err := h.readQueryRequest(r) if err != nil { @@ -337,46 +337,13 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { h.writeQueryResponse(w, r, &QueryResponse{Err: err}) return } + // TODO: Remove + req.Index = mux.Vars(r)["index"] - // Build execution options. - opt := &ExecOptions{ - Remote: req.Remote, - ExcludeAttrs: req.ExcludeAttrs, - ExcludeBits: req.ExcludeBits, - } - - // Parse query string. - q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() + resp, err := h.api.ExecuteQuery(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - - // Execute the query. - results, err := h.Executor.Execute(r.Context(), indexName, q, req.Slices, opt) - resp := &QueryResponse{Results: results, Err: err} - - // Fill column attributes if requested. - if req.ColumnAttrs && !req.ExcludeBits { - // Consolidate all column ids across all calls. - var columnIDs []uint64 - for _, result := range results { - bm, ok := result.(*Bitmap) - if !ok { - continue - } - columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) - } - - // Retrieve column attributes across all calls. - columnAttrSets, err := h.readColumnAttrSets(h.Holder.Index(indexName), columnIDs) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - resp.ColumnAttrSets = columnAttrSets + h.writeQueryResponse(w, r, &resp) } // Set appropriate status code, if there is an error. @@ -390,7 +357,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } // Write response back to client. - if err := h.writeQueryResponse(w, r, resp); err != nil { + if err := h.writeQueryResponse(w, r, &resp); err != nil { h.Logger.Printf("write query response error: %s", err) } } diff --git a/server.go b/server.go index 08923f13a..ede265e19 100644 --- a/server.go +++ b/server.go @@ -117,6 +117,10 @@ func NewServer() *Server { s.Handler.Holder = s.Holder s.diagnostics.server = s + s.Handler.api = &API{ + holder: s.Holder, + } + return s } @@ -172,6 +176,7 @@ func (s *Server) Open() error { s.Handler.Executor = e s.Cluster.prefect = s.Handler + s.Handler.api.executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster From 2ebc1914be0865b074520cda3bfd0a21e897d8c7 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 2 Mar 2018 18:40:50 +0300 Subject: [PATCH 02/79] More API functions --- api.go | 133 ++++++++++++++++++++++++++++++++++++++++++++++++----- handler.go | 112 +++++++++++--------------------------------- server.go | 8 ++-- 3 files changed, 150 insertions(+), 103 deletions(-) diff --git a/api.go b/api.go index 5ab4a36aa..e004821fb 100644 --- a/api.go +++ b/api.go @@ -16,23 +16,31 @@ package pilosa import ( "context" + "fmt" + "io/ioutil" + "log" "strings" + "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) -type QueryOptions struct { - Remote bool - ExcludeAttrs bool - ExcludeBits bool - ColumnAttrs bool +type API struct { + Holder *Holder + // The execution engine for running queries. + Executor interface { + Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) + } + Broadcaster Broadcaster + logger *log.Logger } -type API struct { - holder *Holder - // The execution engine for running queries. - executor interface { - Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) +func NewAPI(logger *log.Logger) *API { + if logger == nil { + logger = log.New(ioutil.Discard, "", 0) + } + return &API{ + logger: logger, } } @@ -49,7 +57,7 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons ExcludeAttrs: req.ExcludeAttrs, ExcludeBits: req.ExcludeBits, } - results, err := a.executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + results, err := a.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) if err != nil { return resp, err } @@ -68,7 +76,7 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons } // Retrieve column attributes across all calls. - columnAttrSets, err := a.readColumnAttrSets(a.holder.Index(req.Index), columnIDs) + columnAttrSets, err := a.readColumnAttrSets(a.Holder.Index(req.Index), columnIDs) if err != nil { return resp, err } @@ -99,3 +107,104 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet return ax, nil } + +func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { + // Create index. + index, err := api.Holder.CreateIndex(indexName, options) + if err != nil { + return nil, err + } + // Send the create index message to all nodes. + err = api.Broadcaster.SendSync( + &internal.CreateIndexMessage{ + Index: indexName, + Meta: options.Encode(), + }) + if err != nil { + api.logger.Printf("problem sending CreateIndex message: %s", err) + return nil, err + } + api.Holder.Stats.Count("createIndex", 1, 1.0) + return index, nil +} + +func (api *API) ReadIndex(ctx context.Context, indexName string) (*Index, error) { + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + return index, nil +} + +func (api *API) DeleteIndex(ctx context.Context, indexName string) error { + // Delete index from the holder. + err := api.Holder.DeleteIndex(indexName) + if err != nil { + return err + } + // Send the delete index message to all nodes. + err = api.Broadcaster.SendSync( + &internal.DeleteIndexMessage{ + Index: indexName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteIndex message: %s", err) + return err + } + api.Holder.Stats.Count("deleteIndex", 1, 1.0) + return nil +} + +func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + // Create frame. + frame, err := index.CreateFrame(frameName, options) + if err != nil { + return nil, err + } + + // Send the create frame message to all nodes. + err = api.Broadcaster.SendSync( + &internal.CreateFrameMessage{ + Index: indexName, + Frame: frameName, + Meta: options.Encode(), + }) + if err != nil { + api.logger.Printf("problem sending CreateFrame message: %s", err) + return nil, err + } + api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + return frame, nil +} + +func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + // Delete frame from the index. + if err := index.DeleteFrame(frameName); err != nil { + return err + } + + // Send the delete frame message to all nodes. + err := api.Broadcaster.SendSync( + &internal.DeleteFrameMessage{ + Index: indexName, + Frame: frameName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteFrame message: %s", err) + return err + } + api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + return nil +} diff --git a/handler.go b/handler.go index c40148b43..bd47524ed 100644 --- a/handler.go +++ b/handler.go @@ -70,7 +70,7 @@ type Handler struct { // Keeps the query argument validators for each handler validators map[string]*queryValidationSpec - api *API + API *API } // externalPrefixFlag denotes endpoints that are intended to be exposed to clients. @@ -340,7 +340,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // TODO: Remove req.Index = mux.Vars(r)["index"] - resp, err := h.api.ExecuteQuery(r.Context(), req) + resp, err := h.API.ExecuteQuery(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) h.writeQueryResponse(w, r, &resp) @@ -385,9 +385,9 @@ func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { // handleGetIndex handles GET /index/ requests. func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + index, err := h.API.ReadIndex(r.Context(), indexName) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) return } @@ -469,28 +469,17 @@ type postIndexResponse struct{} // handleDeleteIndex handles DELETE /index request. func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - - // Delete index from the holder. - if err := h.Holder.DeleteIndex(indexName); err != nil { + err := h.API.DeleteIndex(r.Context(), indexName) + if err != nil { + h.Logger.Printf("problem deleting index: %s", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Send the delete index message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteIndexMessage{ - Index: indexName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteIndex message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.Count("deleteIndex", 1, 1.0) } type deleteIndexResponse struct{} @@ -510,8 +499,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - // Create index. - _, err = h.Holder.CreateIndex(indexName, req.Options) + _, err = h.API.CreateIndex(r.Context(), indexName, req.Options) if err == ErrIndexExists { http.Error(w, err.Error(), http.StatusConflict) return @@ -520,24 +508,10 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { return } - // Send the create index message to all nodes. - err = h.Broadcaster.SendSync( - &internal.CreateIndexMessage{ - Index: indexName, - Meta: req.Options.Encode(), - }) - if err != nil { - h.Logger.Printf("problem sending CreateIndex message: %s", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - // Encode response. if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.Count("createIndex", 1, 1.0) } // handlePatchIndexTimeQuantum handles PATCH /index/time_quantum request. @@ -655,41 +629,22 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Create frame. - _, err = index.CreateFrame(frameName, req.Options) - if err == ErrFrameExists { - http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Send the create frame message to all nodes. - err = h.Broadcaster.SendSync( - &internal.CreateFrameMessage{ - Index: indexName, - Frame: frameName, - Meta: req.Options.Encode(), - }) + _, err = h.API.CreateFrame(r.Context(), indexName, frameName, req.Options) if err != nil { - h.Logger.Printf("problem sending CreateFrame message: %s", err) + switch err { + case ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrFrameExists: + http.Error(w, err.Error(), http.StatusConflict) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return } - // Encode response. if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) } type _postFrameRequest postFrameRequest @@ -742,37 +697,22 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) + err := h.API.DeleteFrame(r.Context(), indexName, frameName) + if err != nil { + if err == ErrIndexNotFound { + if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil { + h.Logger.Printf("response encoding error: %s", err) + } + return } - return - } - - // Delete frame from the index. - if err := index.DeleteFrame(frameName); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Send the delete frame message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteFrameMessage{ - Index: indexName, - Frame: frameName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteFrame message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } - - h.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) } type deleteFrameResponse struct{} diff --git a/server.go b/server.go index ede265e19..1973f746e 100644 --- a/server.go +++ b/server.go @@ -117,10 +117,7 @@ func NewServer() *Server { s.Handler.Holder = s.Holder s.diagnostics.server = s - s.Handler.api = &API{ - holder: s.Holder, - } - + s.Handler.API = NewAPI(s.logger) return s } @@ -169,6 +166,7 @@ func (s *Server) Open() error { // Initialize HTTP handler. s.Handler.Broadcaster = s.Broadcaster + s.Handler.API.Broadcaster = s.Broadcaster s.Handler.BroadcastHandler = s s.Handler.StatusHandler = s s.Handler.Node = node @@ -176,7 +174,7 @@ func (s *Server) Open() error { s.Handler.Executor = e s.Cluster.prefect = s.Handler - s.Handler.api.executor = e + s.Handler.API.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster From b702f70610962341116d9678ce7c5877b7e171ca Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 5 Mar 2018 18:28:24 +0300 Subject: [PATCH 03/79] Moved more of handler to API --- api.go | 464 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 462 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index e004821fb..e544d2d3f 100644 --- a/api.go +++ b/api.go @@ -16,11 +16,17 @@ package pilosa import ( "context" + "encoding/csv" "fmt" + "io" "io/ioutil" "log" + "net/http" + "reflect" + "strconv" "strings" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" ) @@ -31,8 +37,13 @@ type API struct { Executor interface { Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) } - Broadcaster Broadcaster - logger *log.Logger + Broadcaster Broadcaster + BroadcastHandler BroadcastHandler + StatusHandler StatusHandler + Cluster *Cluster + URI *URI + RemoteClient *http.Client + logger *log.Logger } func NewAPI(logger *log.Logger) *API { @@ -208,3 +219,452 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) return nil } + +func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { + // Find the fragment. + f := api.Holder.Fragment(indexName, frameName, viewName, slice) + if f == nil { + return ErrFragmentNotFound + } + + // Wrap writer with a CSV writer. + cw := csv.NewWriter(w) + + // Iterate over each bit. + if err := f.ForEachBit(func(rowID, columnID uint64) error { + return cw.Write([]string{ + strconv.FormatUint(rowID, 10), + strconv.FormatUint(columnID, 10), + }) + }); err != nil { + return err + } + + // Ensure data is flushed. + cw.Flush() + + return nil +} + +func (api *API) FragmentNodes(ctx context.Context, indexName string, slice uint64) []*Node { + return api.Cluster.FragmentNodes(indexName, slice) +} + +func (api *API) FragmentData(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (*Fragment, error) { + // Retrieve fragment from holder. + f := api.Holder.Fragment(indexName, frameName, viewName, slice) + if f == nil { + return nil, ErrFragmentNotFound + } + return f, nil +} + +func (api *API) WriteFragmentData(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { + // Retrieve frame. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Retrieve view. + view, err := f.CreateViewIfNotExists(viewName) + if err != nil { + return err + } + + // Retrieve fragment from frame. + frag, err := view.CreateFragmentIfNotExists(slice) + if err != nil { + return err + } + + // Read fragment in from request body. + if _, err := frag.ReadFrom(reader); err != nil { + return err + } + return nil +} + +func (api *API) FragmentBlockData(ctx context.Context, req internal.BlockDataRequest) (internal.BlockDataResponse, error) { + // Retrieve fragment from holder. + f := api.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) + if f == nil { + return internal.BlockDataResponse{}, ErrFragmentNotFound + } + + // Read data + var resp internal.BlockDataResponse + resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) + return resp, nil +} + +func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { + // Retrieve fragment from holder. + f := api.Holder.Fragment(indexName, frameName, viewName, slice) + if f == nil { + return nil, ErrFragmentNotFound + } + + // Retrieve blocks. + blocks := f.Blocks() + return blocks, nil +} + +func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { + // Create a client for the remote cluster. + client := NewInternalHTTPClientFromURI(host, api.RemoteClient) + + // Determine the maximum number of slices. + maxSlices, err := client.MaxSliceByIndex(ctx) + if err != nil { + return err + } + + // Retrieve frame. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Retrieve list of all views. + views, err := client.FrameViews(ctx, indexName, frameName) + if err != nil { + return err + } + + // Loop over each slice and import it if this node owns it. + for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { + // Ignore this slice if we don't own it. + if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + continue + } + + // Loop over view names. + for _, view := range views { + // Create view. + v, err := f.CreateViewIfNotExists(view) + if err != nil { + return err + } + + // Otherwise retrieve the local fragment. + frag, err := v.CreateFragmentIfNotExists(slice) + if err != nil { + return err + } + + // Stream backup from remote node. + rd, err := client.BackupSlice(ctx, indexName, frameName, view, slice) + if err != nil { + return err + } else if rd == nil { + continue // slice doesn't exist + } + + // Restore to local frame and always close reader. + if err := func() error { + defer rd.Close() + if _, err := frag.ReadFrom(rd); err != nil { + return err + } + return nil + }(); err != nil { + return err + } + } + } + + return nil +} + +func (api *API) ClusterHosts(ctx context.Context) []*Node { + return api.Cluster.Nodes +} + +func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + if err := inputDef.Validate(); err != nil { + return err + } + + // Encode InputDefinition to its internal representation. + def := inputDef.Encode() + def.Name = inputDefName + + // Create InputDefinition. + if _, err := index.CreateInputDefinition(def); err != nil { + return err + } + + err := api.Broadcaster.SendSync( + &internal.CreateInputDefinitionMessage{ + Index: indexName, + Definition: def, + }) + if err != nil { + api.logger.Printf("problem sending CreateInputDefinition message: %s", err) + } + return nil +} + +func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + inputDef, err := index.InputDefinition(inputDefName) + if err != nil { + return nil, err + } + return inputDef, nil +} + +func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + // Delete input definition from the index. + if err := index.DeleteInputDefinition(inputDefName); err != nil { + return err + } + + err := api.Broadcaster.SendSync( + &internal.DeleteInputDefinitionMessage{ + Index: indexName, + Name: inputDefName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteInputDefinition message: %s", err) + } + return nil +} + +func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { + // Find index. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + for _, req := range reqs { + bits, err := api.inputJSONDataParser(req.(map[string]interface{}), index, inputDefName) + if err != nil { + return err + } + for fr, bs := range bits { + if err := index.InputBits(fr, bs); err != nil { + return err + } + } + } + + return nil +} + +func (api *API) RecalculateCaches(ctx context.Context) { + api.Holder.RecalculateCaches() +} + +func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error { + // Forward the error message. + if err := api.BroadcastHandler.ReceiveMessage(pb); err != nil { + return err + } + return nil +} + +func (api *API) LocalID(ctx context.Context) string { + return api.Holder.LocalID +} + +func (api *API) Schema(ctx context.Context) []*IndexInfo { + return api.Holder.Schema() +} + +func (api *API) Status(ctx context.Context) (proto.Message, error) { + return api.StatusHandler.ClusterStatus() +} + +func (api *API) CreateFrameField(ctx context.Context, indexName string, frameName string, field *Field) error { + // Retrieve frame by name. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Create new field. + if err := f.CreateField(field); err != nil { + return err + } + + // Send the create field message to all nodes. + err := api.Broadcaster.SendSync( + &internal.CreateFieldMessage{ + Index: indexName, + Frame: frameName, + Field: encodeField(field), + }) + if err != nil { + api.logger.Printf("problem sending CreateField message: %s", err) + } + return err +} + +func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameName string, fieldName string) error { + // Retrieve frame by name. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Delete field. + if err := f.DeleteField(fieldName); err != nil { + return err + } + + // Send the delete field message to all nodes. + err := api.Broadcaster.SendSync( + &internal.DeleteFieldMessage{ + Index: indexName, + Frame: frameName, + Field: fieldName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteField message: %s", err) + } + return err +} + +func (api *API) FrameFields(ctx context.Context, indexName string, frameName string) (*FrameSchema, error) { + index := api.Holder.index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + frame := index.frame(frameName) + if frame == nil { + return nil, ErrFrameNotFound + } + + return frame.GetFields() +} + +func (api *API) FrameViews(ctx context.Context, indexName string, frameName string) ([]*View, error) { + // Retrieve views. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return nil, ErrFrameNotFound + } + + // Fetch views. + views := f.Views() + return views, nil +} + +func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { + // Retrieve frame. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return ErrFrameNotFound + } + + // Delete the view. + if err := f.DeleteView(viewName); err != nil { + // Ingore this error becuase views do not exist on all nodes due to slice distribution. + if err != ErrInvalidView { + return err + } + } + + // Send the delete view message to all nodes. + err := api.Broadcaster.SendSync( + &internal.DeleteViewMessage{ + Index: indexName, + Frame: frameName, + View: viewName, + }) + if err != nil { + api.logger.Printf("problem sending DeleteView message: %s", err) + } + + return err +} + +// InputJSONDataParser validates input json file and executes SetBit. +func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { + inputDef, err := index.InputDefinition(name) + if err != nil { + return nil, err + } + // If field in input data is not in defined definition, return error. + var colValue uint64 + validFields := make(map[string]bool) + timestampFrame := make(map[string]int64) + for _, field := range inputDef.Fields() { + validFields[field.Name] = true + if field.PrimaryKey { + value, ok := req[field.Name] + if !ok { + return nil, fmt.Errorf("primary key does not exist") + } + rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float + if !ok { + return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) + } + colValue = uint64(rawValue) + } + // Find frame that need to add timestamp. + for _, action := range field.Actions { + if action.ValueDestination == InputSetTimestamp { + timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) + if err != nil { + return nil, err + } + } + } + } + + for key := range req { + _, ok := validFields[key] + if !ok { + return nil, fmt.Errorf("field not found: %s", key) + } + } + + setBits := make(map[string][]*Bit) + + for _, field := range inputDef.Fields() { + // skip field that defined in definition but not in input data + if _, ok := req[field.Name]; !ok { + continue + } + + // Looking into timestampFrame map and set timestamp to the whole frame + for _, action := range field.Actions { + frame := action.Frame + timestamp := timestampFrame[action.Frame] + // Skip input data field values that are set to null + if req[field.Name] == nil { + continue + } + bit, err := HandleAction(action, req[field.Name], colValue, timestamp) + if err != nil { + return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) + } + if bit != nil { + setBits[frame] = append(setBits[frame], bit) + } + } + } + return setBits, nil +} From 72526f753ce0d05033df6e1dbade86d291212fca Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 5 Mar 2018 18:28:24 +0300 Subject: [PATCH 04/79] Moved more of handler to API --- api.go | 2 +- handler.go | 478 ++++++++++++++++------------------------------------- 2 files changed, 141 insertions(+), 339 deletions(-) diff --git a/api.go b/api.go index e544d2d3f..84c2f5146 100644 --- a/api.go +++ b/api.go @@ -484,7 +484,7 @@ func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error } func (api *API) LocalID(ctx context.Context) string { - return api.Holder.LocalID + return api.Cluster.Node.ID } func (api *API) Schema(ctx context.Context) []*IndexInfo { diff --git a/handler.go b/handler.go index bd47524ed..f9fc73c52 100644 --- a/handler.go +++ b/handler.go @@ -16,7 +16,6 @@ package pilosa import ( "context" - "encoding/csv" "encoding/json" "errors" "expvar" @@ -44,21 +43,7 @@ import ( // Handler represents an HTTP handler. type Handler struct { - Holder *Holder - Broadcaster Broadcaster - BroadcastHandler BroadcastHandler - StatusHandler StatusHandler - - FileSystem FileSystem - - // Local hostname & cluster configuration. - Node *Node - Cluster *Cluster - RemoteClient *http.Client - - Router *mux.Router - NormalRouter *mux.Router - RestrictedRouter *mux.Router + Router *mux.Router // The execution engine for running queries. Executor interface { @@ -295,8 +280,9 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { // handleGetSchema handles GET /schema requests. func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { + schema := h.API.Schema(r.Context()) if err := json.NewEncoder(w).Encode(getSchemaResponse{ - Indexes: h.Holder.Schema(), + Indexes: schema, }); err != nil { h.Logger.Printf("write schema response error: %s", err) } @@ -304,7 +290,7 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - pb, err := h.StatusHandler.ClusterStatus() + status, err := h.API.Status(r.Context()) if err != nil { h.Logger.Printf("cluster status error: %s", err) return @@ -774,13 +760,6 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { return } - // Retrieve frame by name. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - field := &Field{ Name: fieldName, Type: req.Type, @@ -788,23 +767,15 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { Max: req.Max, } - // Create new field. - if err := f.CreateField(field); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if err := h.API.CreateFrameField(r.Context(), indexName, frameName, field); err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Send the create field message to all nodes. - err := h.Broadcaster.SendSync( - &internal.CreateFieldMessage{ - Index: indexName, - Frame: frameName, - Field: encodeField(field), - }) - if err != nil { - h.Logger.Printf("problem sending CreateField message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(postFrameFieldResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) @@ -825,30 +796,15 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) frameName := mux.Vars(r)["frame"] fieldName := mux.Vars(r)["field"] - // Retrieve frame by name. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + if err := h.API.DeleteFrameField(r.Context(), indexName, frameName, fieldName); err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Delete field. - if err := f.DeleteField(fieldName); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Send the delete field message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteFieldMessage{ - Index: indexName, - Frame: frameName, - Field: fieldName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteField message: %s", err) - } - // Encode response. if err := json.NewEncoder(w).Encode(deleteFrameFieldResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) @@ -859,24 +815,18 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - index := h.Holder.index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - frame := index.frame(frameName) - if frame == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - fields, err := frame.GetFields() - if err == ErrFrameFieldsNotAllowed { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } else if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + schema, err := h.API.FrameFields(r.Context(), indexName, frameName) + if err != nil { + switch err { + case ErrIndexNotFound: + fallthrough + case ErrFrameNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrFrameFieldsNotAllowed: + http.Error(w, err.Error(), http.StatusBadRequest) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -899,15 +849,16 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - // Retrieve views. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + views, err := h.API.FrameViews(r.Context(), indexName, frameName) + if err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Fetch views. - views := f.Views() names := make([]string, len(views)) for i := range views { names[i] = views[i].Name() @@ -925,31 +876,13 @@ func (h *Handler) handleDeleteView(w http.ResponseWriter, r *http.Request) { frameName := mux.Vars(r)["frame"] viewName := mux.Vars(r)["view"] - // Retrieve frame. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Delete the view. - if err := f.DeleteView(viewName); err != nil { - // Ingore this error because views do not exist on all nodes due to slice distribution. - if err != ErrInvalidView { + if err := h.API.DeleteView(r.Context(), indexName, frameName, viewName); err != nil { + if err == ErrFrameNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { http.Error(w, err.Error(), http.StatusBadRequest) - return } - } - - // Send the delete view message to all nodes. - err := h.Broadcaster.SendSync( - &internal.DeleteViewMessage{ - Index: indexName, - Frame: frameName, - View: viewName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteView message: %s", err) + return } // Encode response. @@ -1314,28 +1247,13 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { return } - // Find the fragment. - f := h.Holder.Fragment(index, frame, view, slice) - if f == nil { - return + if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil { + if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } - - // Wrap writer with a CSV writer. - cw := csv.NewWriter(w) - - // Iterate over each bit. - if err := f.ForEachBit(func(rowID, columnID uint64) error { - return cw.Write([]string{ - strconv.FormatUint(rowID, 10), - strconv.FormatUint(columnID, 10), - }) - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Ensure data is flushed. - cw.Flush() } // handleGetFragmentNodes handles /fragment/nodes requests. @@ -1351,7 +1269,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } // Retrieve fragment owner nodes. - nodes := h.Cluster.FragmentNodes(index, slice) + nodes := h.API.FragmentNodes(r.Context(), index, slice) // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { @@ -1370,9 +1288,9 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve fragment from holder. - f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice) - if f == nil { - http.Error(w, "fragment not found", http.StatusNotFound) + f, err := h.API.FragmentData(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) return } @@ -1392,31 +1310,12 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) return } - // Retrieve frame. - f := h.Holder.Frame(q.Get("index"), q.Get("frame")) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve view. - view, err := f.CreateViewIfNotExists(q.Get("view")) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Retrieve fragment from frame. - frag, err := view.CreateFragmentIfNotExists(slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Read fragment in from request body. - if _, err := frag.ReadFrom(r.Body); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + if err = h.API.WriteFragmentData(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil { + if err == ErrFrameNotFound { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } } @@ -1432,19 +1331,16 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ return } - // Retrieve fragment from holder. - f := h.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) - if f == nil { - http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) + resp, err := h.API.FragmentBlockData(r.Context(), req) + if err != nil { + if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Read data - var resp internal.BlockDataResponse - if f != nil { - resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) - } - // Encode response. buf, err := proto.Marshal(&resp) if err != nil { @@ -1468,16 +1364,16 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request return } - // Retrieve fragment from holder. - f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice) - if f == nil { - http.Error(w, "fragment not found", http.StatusNotFound) + blocks, err := h.API.FragmentBlocks(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) + if err != nil { + if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } - // Retrieve blocks. - blocks := f.Blocks() - // Encode response. if err := json.NewEncoder(w).Encode(getFragmentBlocksResponse{ Blocks: blocks, @@ -1509,80 +1405,23 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) http.Error(w, err.Error(), http.StatusBadRequest) } - // Create a client for the remote cluster. - client := NewInternalHTTPClientFromURI(host, h.RemoteClient) - - // Determine the maximum number of slices. - maxSlices, err := client.MaxSliceByIndex(r.Context()) - if err != nil { - http.Error(w, "cannot determine remote slice count: "+err.Error(), http.StatusInternalServerError) - return - } - - // Retrieve frame. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve list of all views. - views, err := client.FrameViews(r.Context(), indexName, frameName) - if err != nil { - http.Error(w, "cannot retrieve frame views: "+err.Error(), http.StatusInternalServerError) - return - } - - // Loop over each slice and import it if this node owns it. - for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { - // Ignore this slice if we don't own it. - if !h.Cluster.OwnsFragment(h.Node.ID, indexName, slice) { - continue - } - - // Loop over view names. - for _, view := range views { - // Create view. - v, err := f.CreateViewIfNotExists(view) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Otherwise retrieve the local fragment. - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Stream backup from remote node. - rd, err := client.BackupSlice(r.Context(), indexName, frameName, view, slice) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } else if rd == nil { - continue // slice doesn't exist - } - - // Restore to local frame and always close reader. - if err := func() error { - defer rd.Close() - if _, err := frag.ReadFrom(rd); err != nil { - return err - } - return nil - }(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } + err = h.API.RestoreFrame(r.Context(), indexName, frameName, host) + switch err { + case nil: + break + case ErrFrameNotFound: + fallthrough + case ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) } } // handleGetHosts handles /hosts requests. func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { - if err := json.NewEncoder(w).Encode(h.Cluster.Nodes); err != nil { + hosts := h.API.ClusterHosts(r.Context()) + if err := json.NewEncoder(w).Encode(hosts); err != nil { h.Logger.Printf("write version response error: %s", err) } } @@ -1766,13 +1605,6 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - // Decode request. var req InputDefinitionInfo err := json.NewDecoder(r.Body).Decode(&req) @@ -1781,36 +1613,26 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - if err := req.Validate(); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Encode InputDefinition to its internal representation. - def := req.Encode() - def.Name = inputDefName - - // Create InputDefinition. - _, err = index.CreateInputDefinition(def) - if err == ErrInputDefinitionExists { + err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req) + switch err { + case nil: + break + case ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrInputDefinitionExists: http.Error(w, err.Error(), http.StatusConflict) - return - } else if err != nil { + case ErrInputDefinitionAttrsRequired: + fallthrough + case ErrInputDefinitionNameRequired: + fallthrough + case ErrInputDefinitionActionRequired: + fallthrough + case ErrInputDefinitionHasPrimaryKey: + fallthrough + case ErrInputDefinitionDupePrimaryKey: + http.Error(w, err.Error(), http.StatusBadRequest) + default: http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - err = h.Broadcaster.SendSync( - &internal.CreateInputDefinitionMessage{ - Index: indexName, - Definition: def, - }) - if err != nil { - h.Logger.Printf("problem sending CreateInputDefinition message: %s", err) - } - - if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) } } @@ -1819,16 +1641,18 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - inputDef, err := index.InputDefinition(inputDefName) + inputDef, err := h.API.InputDefinition(r.Context(), indexName, inputDefName) if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) + switch err { + case nil: + break + case ErrIndexNotFound: + fallthrough + case ErrInputDefinitionNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -1838,7 +1662,6 @@ func (h *Handler) handleGetInputDefinition(w http.ResponseWriter, r *http.Reques }); err != nil { h.Logger.Printf("write status response error: %s", err) } - } // handleDeleteInputDefinition handles DELETE /input-definition request. @@ -1846,28 +1669,20 @@ func (h *Handler) handleDeleteInputDefinition(w http.ResponseWriter, r *http.Req indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) + if err := h.API.DeleteInputDefinition(r.Context(), indexName, inputDefName); err != nil { + switch err { + case nil: + break + case ErrIndexNotFound: + fallthrough + case ErrInputDefinitionNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: + http.Error(w, err.Error(), http.StatusNotFound) + } return } - // Delete input definition from the index. - if err := index.DeleteInputDefinition(inputDefName); err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - err := h.Broadcaster.SendSync( - &internal.DeleteInputDefinitionMessage{ - Index: indexName, - Name: inputDefName, - }) - if err != nil { - h.Logger.Printf("problem sending DeleteInputDefinition message: %s", err) - } - if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } @@ -1879,13 +1694,6 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] inputDefName := mux.Vars(r)["input-definition"] - // Find index. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - // Decode request. var reqs []interface{} err := json.NewDecoder(r.Body).Decode(&reqs) @@ -1893,28 +1701,27 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - for _, req := range reqs { - bits, err := h.InputJSONDataParser(req.(map[string]interface{}), index, inputDefName) - if err == ErrInputDefinitionNotFound { + + if err = h.API.WriteInput(r.Context(), indexName, inputDefName, reqs); err != nil { + switch err { + case nil: + break + case ErrIndexNotFound: + fallthrough + case ErrInputDefinitionNotFound: http.Error(w, err.Error(), http.StatusNotFound) - return - } else if err != nil { + default: http.Error(w, err.Error(), http.StatusBadRequest) - return - } - for fr, bs := range bits { - err := index.InputBits(fr, bs) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } } + return } + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { h.Logger.Printf("response encoding error: %s", err) } } +// <<<<<<< b702f70610962341116d9678ce7c5877b7e171ca // handlePostClusterResizeSetCoordinator handles POST /cluster/resize/set-coordinator request. func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -2109,14 +1916,11 @@ func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, return setBits, nil } +// ======= +// >>>>>>> Moved more of handler to API + func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - err := h.Broadcaster.SendSync(&internal.RecalculateCaches{}) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) - return - } - h.Holder.RecalculateCaches() + h.API.RecalculateCaches(r.Context()) w.WriteHeader(http.StatusNoContent) } @@ -2161,9 +1965,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques return } - // Forward the error message. - err = h.BroadcastHandler.ReceiveMessage(pb) - if err != nil { + if err := h.API.PostClusterMessage(r.Context(), pb); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -2174,7 +1976,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte(h.Cluster.Node.ID)) + _, err := w.Write([]byte(h.API.LocalID(r.Context()))) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } From 8336784d57d48827cf60787ce5d431f1c2a56d93 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 6 Mar 2018 17:28:33 +0300 Subject: [PATCH 05/79] More API updates; removed Cluster, Holder, etc from Handler --- api.go | 168 ++++++++++++++++++++++++++++++- handler.go | 255 ++++++++++++++++------------------------------- pilosa.go | 4 + server.go | 12 +-- server/server.go | 3 +- 5 files changed, 262 insertions(+), 180 deletions(-) diff --git a/api.go b/api.go index 84c2f5146..f369ba10a 100644 --- a/api.go +++ b/api.go @@ -25,6 +25,7 @@ import ( "reflect" "strconv" "strings" + "time" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" @@ -221,6 +222,12 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str } func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { + // Validate that this handler owns the slice. + if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + api.logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + return ErrClusterDoesNotOwnSlice + } + // Find the fragment. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -600,7 +607,166 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri return err } -// InputJSONDataParser validates input json file and executes SetBit. +func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + // Retrieve index from holder. + index := api.Holder.Index(indexName) + if index == nil { + return nil, ErrIndexNotFound + } + + // Retrieve local blocks. + localBlocks, err := index.ColumnAttrStore().Blocks() + if err != nil { + return nil, err + } + + // Read all attributes from all mismatched blocks. + attrs := make(map[uint64]map[string]interface{}) + for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + // Retrieve block data. + m, err := index.ColumnAttrStore().BlockData(blockID) + if err != nil { + return nil, err + } + + // Copy to index-wide struct. + for k, v := range m { + attrs[k] = v + } + } + return attrs, nil +} + +func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + // Retrieve index from holder. + f := api.Holder.Frame(indexName, frameName) + if f == nil { + return nil, ErrFrameNotFound + } + + // Retrieve local blocks. + localBlocks, err := f.RowAttrStore().Blocks() + if err != nil { + return nil, err + } + + // Read all attributes from all mismatched blocks. + attrs := make(map[uint64]map[string]interface{}) + for _, blockID := range AttrBlocks(localBlocks).Diff(blocks) { + // Retrieve block data. + m, err := f.RowAttrStore().BlockData(blockID) + if err != nil { + return nil, err + } + + // Copy to index-wide struct. + for k, v := range m { + attrs[k] = v + } + } + return attrs, nil +} + +func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + if err != nil { + return err + } + + // Convert timestamps to time.Time. + timestamps := make([]*time.Time, len(req.Timestamps)) + for i, ts := range req.Timestamps { + if ts == 0 { + continue + } + t := time.Unix(0, ts) + timestamps[i] = &t + } + + // Import into fragment. + err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) + if err != nil { + api.logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + } + return err +} + +func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) + if err != nil { + return err + } + + // Import into fragment. + err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) + if err != nil { + api.logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + } + return err +} + +func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { + // Retrieve index by name. + index := api.Holder.Index(indexName) + if index == nil { + return ErrIndexNotFound + } + + // Set default time quantum on index. + return index.SetTimeQuantum(timeQuantum) +} + +func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { + // Retrieve index by name. + frame := api.Holder.Frame(indexName, frameName) + if frame == nil { + return ErrFrameNotFound + } + + // Set default time quantum on index. + return frame.SetTimeQuantum(timeQuantum) +} + +func (api *API) SliceMax(ctx context.Context, inverse bool) map[string]uint64 { + if inverse { + return api.Holder.MaxInverseSlices() + } + return api.Holder.MaxSlices() +} + +func (api *API) StatsWithTags(tags []string) StatsClient { + return api.Holder.Stats.WithTags(tags...) +} + +func (api *API) ClusterLongQueryTime() time.Duration { + return api.Cluster.LongQueryTime +} + +func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { + // Validate that this handler owns the slice. + if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + api.logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + return nil, nil, ErrClusterDoesNotOwnSlice + } + + // Find the Index. + api.logger.Println("importing:", indexName, frameName, slice) + index := api.Holder.Index(indexName) + if index == nil { + api.logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) + return nil, nil, ErrIndexNotFound + } + + // Retrieve frame. + frame := index.Frame(frameName) + if frame == nil { + api.logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) + return nil, nil, ErrFrameNotFound + } + return index, frame, nil +} + +// inputJSONDataParser validates input json file and executes SetBit. func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef, err := index.InputDefinition(name) if err != nil { diff --git a/handler.go b/handler.go index f9fc73c52..0f04028e0 100644 --- a/handler.go +++ b/handler.go @@ -240,27 +240,26 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { dif := time.Since(t) // Calculate per request StatsD metrics when the handler is fully configured. - if h.Holder != nil && h.Cluster != nil { - statsTags := make([]string, 0, 3) + statsTags := make([]string, 0, 3) - if h.Cluster.LongQueryTime > 0 && dif > h.Cluster.LongQueryTime { - h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) - statsTags = append(statsTags, "slow_query") - } - - pathParts := strings.Split(r.URL.Path, "/") - endpointName := strings.Join(pathParts, "_") - - if externalPrefixFlag[pathParts[1]] { - statsTags = append(statsTags, "external") - } - - // useragent tag identifies internal/external endpoints - statsTags = append(statsTags, "useragent:"+r.UserAgent()) - - stats := h.Holder.Stats.WithTags(statsTags...) - stats.Histogram("http."+endpointName, float64(dif), 0.1) + longQueryTime := h.API.ClusterLongQueryTime() + if longQueryTime > 0 && dif > longQueryTime { + h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) + statsTags = append(statsTags, "slow_query") } + + pathParts := strings.Split(r.URL.Path, "/") + endpointName := strings.Join(pathParts, "_") + + if externalPrefixFlag[pathParts[1]] { + statsTags = append(statsTags, "external") + } + + // useragent tag identifies internal/external endpoints + statsTags = append(statsTags, "useragent:"+r.UserAgent()) + + stats := h.API.StatsWithTags(statsTags) + stats.Histogram("http."+endpointName, float64(dif), 0.1) } func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { @@ -348,13 +347,23 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -// handleGetSlicesMax handles GET /schema requests. -func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { - if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ - Standard: h.Holder.MaxSlices(), - Inverse: h.Holder.MaxInverseSlices(), - }); err != nil { - h.Logger.Printf("write slices-max response error: %s", err) +func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { + inverse, err := strconv.ParseBool(r.URL.Query().Get("inverse")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + ms := h.API.SliceMax(r.Context(), inverse) + if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { + pb := &internal.MaxSlicesResponse{ + MaxSlices: ms, + } + if buf, err := proto.Marshal(pb); err != nil { + h.Logger.Printf("protobuf marshal error: %s", err) + } else if _, err := w.Write(buf); err != nil { + h.Logger.Printf("stream write error: %s", err) + } + return } } @@ -518,16 +527,12 @@ func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Req return } - // Retrieve index by name. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Set default time quantum on index. - if err := index.SetTimeQuantum(tq); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if err = h.API.ModifyIndexTimeQuantum(r.Context(), indexName, tq); err != nil { + if err == ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -554,34 +559,14 @@ func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request return } - // Retrieve index from holder. - index := h.Holder.Index(indexName) - if index == nil { - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve local blocks. - blks, err := index.ColumnAttrStore().Blocks() + attrs, err := h.API.IndexAttrDiff(r.Context(), indexName, req.Blocks) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Read all attributes from all mismatched blocks. - attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { - // Retrieve block data. - m, err := index.ColumnAttrStore().BlockData(blockID) - if err != nil { + if err == ErrIndexNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Copy to index-wide struct. - for k, v := range m { - attrs[k] = v } + return } // Encode response. @@ -722,16 +707,12 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req return } - // Retrieve index by name. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Set default time quantum on index. - if err := f.SetTimeQuantum(tq); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if err := h.API.ModifyFrameTimeQuantum(r.Context(), indexName, frameName, tq); err != nil { + if err == ErrFragmentNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -909,34 +890,15 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request return } - // Retrieve index from holder. - f := h.Holder.Frame(indexName, frameName) - if f == nil { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve local blocks. - blks, err := f.RowAttrStore().Blocks() + attrs, err := h.API.FrameAttrDiff(r.Context(), indexName, frameName, req.Blocks) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Read all attributes from all mismatched blocks. - attrs := make(map[uint64]map[string]interface{}) - for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { - // Retrieve block data. - m, err := f.RowAttrStore().BlockData(blockID) - if err != nil { + switch err { + case ErrFragmentNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + default: http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Copy to index-wide struct. - for k, v := range m { - attrs[k] = v } + return } // Encode response. @@ -1094,44 +1056,17 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { return } - // Convert timestamps to time.Time. - timestamps := make([]*time.Time, len(req.Timestamps)) - for i, ts := range req.Timestamps { - if ts == 0 { - continue + if err := h.API.Import(r.Context(), req); err != nil { + switch err { + case ErrIndexNotFound: + fallthrough + case ErrFrameNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) } - t := time.Unix(0, ts) - timestamps[i] = &t - } - - // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) { - msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice) - http.Error(w, msg, http.StatusPreconditionFailed) - return - } - - // Find the Index. - h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice) - index := h.Holder.Index(req.Index) - if index == nil { - h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve frame. - f := index.Frame(req.Frame) - if f == nil { - h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Import into fragment. - err = f.Import(req.RowIDs, req.ColumnIDs, timestamps) - if err != nil { - h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) return } @@ -1174,34 +1109,17 @@ func (h *Handler) handlePostImportValue(w http.ResponseWriter, r *http.Request) return } - // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Node.ID, req.Index, req.Slice) { - msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, req.Index, req.Slice) - http.Error(w, msg, http.StatusPreconditionFailed) - return - } - - // Find the Index. - h.Logger.Printf("importing: %s %s %d", req.Index, req.Frame, req.Slice) - index := h.Holder.Index(req.Index) - if index == nil { - h.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error()) - http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound) - return - } - - // Retrieve frame. - f := index.Frame(req.Frame) - if f == nil { - h.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error()) - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - return - } - - // Import into fragment. - err = f.ImportValue(req.Field, req.ColumnIDs, req.Values) - if err != nil { - h.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + if err = h.API.ImportValue(r.Context(), req); err != nil { + switch err { + case ErrIndexNotFound: + fallthrough + case ErrFrameNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } return } @@ -1240,19 +1158,16 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { return } - // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Node.ID, index, slice) { - msg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Node.ID, index, slice) - http.Error(w, msg, http.StatusPreconditionFailed) - return - } - if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil { - if err == ErrFragmentNotFound { + switch err { + case ErrFragmentNotFound: http.Error(w, err.Error(), http.StatusNotFound) - } else { + case ErrClusterDoesNotOwnSlice: + http.Error(w, err.Error(), http.StatusPreconditionFailed) + default: http.Error(w, err.Error(), http.StatusInternalServerError) } + return } } diff --git a/pilosa.go b/pilosa.go index 98c67203f..04c76a133 100644 --- a/pilosa.go +++ b/pilosa.go @@ -74,6 +74,10 @@ var ( ErrTooManyWrites = errors.New("too many write commands") ErrConfigClusterEnabledHosts = errors.New("providing hosts to a non-disabled cluster is not allowed") + ErrConfigClusterTypeInvalid = errors.New("invalid cluster type") + ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts") + + ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") ) // Regular expression to validate index and frame names. diff --git a/server.go b/server.go index 1973f746e..338183cf7 100644 --- a/server.go +++ b/server.go @@ -115,9 +115,8 @@ func NewServer() *Server { Logger: NopLogger, } - s.Handler.Holder = s.Holder - s.diagnostics.server = s s.Handler.API = NewAPI(s.logger) + s.Handler.API.Holder = s.Holder return s } @@ -165,12 +164,11 @@ func (s *Server) Open() error { s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest // Initialize HTTP handler. - s.Handler.Broadcaster = s.Broadcaster s.Handler.API.Broadcaster = s.Broadcaster - s.Handler.BroadcastHandler = s - s.Handler.StatusHandler = s - s.Handler.Node = node - s.Handler.Cluster = s.Cluster + s.Handler.API.BroadcastHandler = s + s.Handler.API.StatusHandler = s + s.Handler.API.URI = s.URI + s.Handler.API.Cluster = s.Cluster s.Handler.Executor = e s.Cluster.prefect = s.Handler diff --git a/server/server.go b/server/server.go index 66725f4ee..a50ad3e71 100644 --- a/server/server.go +++ b/server/server.go @@ -218,8 +218,7 @@ func (m *Command) SetupServer() error { } c := pilosa.GetHTTPClient(TLSConfig) m.Server.RemoteClient = c - m.Server.Handler.RemoteClient = c - m.Server.Cluster.RemoteClient = c + m.Server.Handler.API.RemoteClient = c // Statik file system. m.Server.Handler.FileSystem = &statik.FileSystem{} From c36dd395e664be375d37fe70f9988d5104a20d57 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 7 Mar 2018 15:39:44 +0300 Subject: [PATCH 06/79] All tests pass --- api.go | 7 +- client_test.go | 67 ++++++++++--------- ctl/backup_test.go | 10 ++- ctl/export_test.go | 10 ++- ctl/import_test.go | 21 +++--- ctl/restore_test.go | 10 ++- executor_test.go | 10 +-- handler.go | 67 +++++++++++-------- handler_test.go | 158 ++++++++++++++++++++++---------------------- holder_test.go | 2 +- stats_test.go | 16 ++--- test/handler.go | 33 ++++++--- 12 files changed, 218 insertions(+), 193 deletions(-) diff --git a/api.go b/api.go index f369ba10a..97d7828e5 100644 --- a/api.go +++ b/api.go @@ -61,7 +61,6 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { - // TODO: Wrap return resp, err } execOpts := &ExecOptions{ @@ -735,10 +734,16 @@ func (api *API) SliceMax(ctx context.Context, inverse bool) map[string]uint64 { } func (api *API) StatsWithTags(tags []string) StatsClient { + if api.Holder == nil || api.Cluster == nil { + return nil + } return api.Holder.Stats.WithTags(tags...) } func (api *API) ClusterLongQueryTime() time.Duration { + if api.Cluster == nil { + return 0 + } return api.Cluster.LongQueryTime } diff --git a/client_test.go b/client_test.go index f2bd18e50..36b9dc38d 100644 --- a/client_test.go +++ b/client_test.go @@ -36,10 +36,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { for i := 0; i < numNodes; i++ { hldr[i] = test.MustOpenHolder() server[i] = test.NewServer() - server[i].Handler.Cluster = c - server[i].Handler.Cluster.Nodes[i].URI = server[i].HostURI() - server[i].Handler.Holder = hldr[i].Holder - server[i].Handler.Node = server[i].Handler.Cluster.Nodes[i] + server[i].Handler.API.URI = server[i].HostURI() + server[i].Handler.API.Cluster = c + server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI() + server[i].Handler.API.Holder = hldr[i].Holder } return server, hldr } @@ -86,7 +86,7 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} for i, num := range sliceNums { - owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].HostURI()) + owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].Host()) ownsNum := false for _, ownNum := range owns { if ownNum == num { @@ -217,10 +217,10 @@ func TestClient_Import(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -268,10 +268,10 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -317,10 +317,10 @@ func TestClient_ImportValue(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -355,10 +355,10 @@ func TestClient_BackupRestore(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -420,10 +420,11 @@ func TestClient_BackupInverseView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -457,10 +458,10 @@ func TestClient_BackupInvalidView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -486,10 +487,10 @@ func TestClient_FragmentBlocks(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Retrieve blocks. c := test.MustNewClient(s.Host(), defaultClient) diff --git a/ctl/backup_test.go b/ctl/backup_test.go index d23b1a917..7a4feb3d9 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -50,12 +50,10 @@ func TestBackupCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm := NewBackupCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import.csv") diff --git a/ctl/export_test.go b/ctl/export_test.go index 5d1d5a4d7..2bcccd6ba 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -63,12 +63,10 @@ func TestExportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0] = node - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) diff --git a/ctl/import_test.go b/ctl/import_test.go index 9022efae9..119a56071 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -69,12 +69,10 @@ func TestImportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0] = node - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() cm.Index = "i" @@ -111,16 +109,15 @@ func TestImportCommand_RunValue(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0] = node - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"rangeEnabled": true, "fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) cm.Index = "i" cm.Frame = "f" diff --git a/ctl/restore_test.go b/ctl/restore_test.go index bb8eb3b5f..f1edb7fe9 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -52,12 +52,10 @@ func TestRestoreCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm := NewRestoreCommand(stdin, stdout, stderr) cm.Path = file.Name() diff --git a/executor_test.go b/executor_test.go index c98239137..690bfc5ce 100644 --- a/executor_test.go +++ b/executor_test.go @@ -927,7 +927,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // The local node owns slice 1. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := test.NewExecutor(hldr.Holder, c) @@ -961,7 +961,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create local executor data. The local node owns slice 1. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) @@ -1004,7 +1004,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Create local executor data. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder // Create frame. if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -1056,7 +1056,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create local executor data. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder // Create frame. if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -1130,7 +1130,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Create local executor data on slice 2 & 4. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) diff --git a/handler.go b/handler.go index 0f04028e0..f6a385ccb 100644 --- a/handler.go +++ b/handler.go @@ -257,9 +257,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // useragent tag identifies internal/external endpoints statsTags = append(statsTags, "useragent:"+r.UserAgent()) - stats := h.API.StatsWithTags(statsTags) - stats.Histogram("http."+endpointName, float64(dif), 0.1) + if stats != nil { + stats.Histogram("http."+endpointName, float64(dif), 0.1) + } } func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { @@ -328,7 +329,8 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { resp, err := h.API.ExecuteQuery(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &resp) + h.writeQueryResponse(w, r, &QueryResponse{Err: err}) + return } // Set appropriate status code, if there is an error. @@ -348,10 +350,15 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { - inverse, err := strconv.ParseBool(r.URL.Query().Get("inverse")) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + var err error + inverse := false + inverseStr := r.URL.Query().Get("inverse") + if inverseStr != "" { + inverse, err = strconv.ParseBool(r.URL.Query().Get("inverse")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } } ms := h.API.SliceMax(r.Context(), inverse) if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { @@ -1161,7 +1168,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil { switch err { case ErrFragmentNotFound: - http.Error(w, err.Error(), http.StatusNotFound) + break case ErrClusterDoesNotOwnSlice: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: @@ -1528,26 +1535,30 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req) - switch err { - case nil: - break - case ErrIndexNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrInputDefinitionExists: - http.Error(w, err.Error(), http.StatusConflict) - case ErrInputDefinitionAttrsRequired: - fallthrough - case ErrInputDefinitionNameRequired: - fallthrough - case ErrInputDefinitionActionRequired: - fallthrough - case ErrInputDefinitionHasPrimaryKey: - fallthrough - case ErrInputDefinitionDupePrimaryKey: - http.Error(w, err.Error(), http.StatusBadRequest) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) + if err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req); err != nil { + switch err { + case ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrInputDefinitionExists: + http.Error(w, err.Error(), http.StatusConflict) + case ErrInputDefinitionAttrsRequired: + fallthrough + case ErrInputDefinitionNameRequired: + fallthrough + case ErrInputDefinitionActionRequired: + fallthrough + case ErrInputDefinitionHasPrimaryKey: + fallthrough + case ErrInputDefinitionDupePrimaryKey: + http.Error(w, err.Error(), http.StatusBadRequest) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) } } diff --git a/handler_test.go b/handler_test.go index 1e30f41ad..39bcc7fff 100644 --- a/handler_test.go +++ b/handler_test.go @@ -65,8 +65,8 @@ func TestHandler_NotFound(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) @@ -100,8 +100,8 @@ func TestHandler_Schema(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { @@ -139,9 +139,9 @@ func TestHandler_Status(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) - h.StatusHandler = s + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.StatusHandler = s s.Handler = h w := httptest.NewRecorder() @@ -186,8 +186,8 @@ func TestHandler_MaxSlices(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { @@ -227,8 +227,8 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { @@ -244,8 +244,8 @@ func TestHandler_Query_Args_URL(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) @@ -272,8 +272,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) @@ -312,8 +312,8 @@ func TestHandler_Query_Args_Err(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { @@ -339,8 +339,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -360,8 +360,8 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -390,8 +390,8 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -423,8 +423,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -446,8 +446,8 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -494,8 +494,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -555,8 +555,8 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -579,8 +579,8 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -612,15 +612,15 @@ func TestHandler_Query_Err_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != http.StatusInternalServerError { + if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -633,8 +633,8 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -643,7 +643,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusInternalServerError { + if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -661,8 +661,8 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) if w.Code != http.StatusMethodNotAllowed { @@ -676,8 +676,8 @@ func TestHandler_Query_ErrParse(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { @@ -693,7 +693,7 @@ func TestHandler_Index_Delete(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -733,8 +733,8 @@ func TestHandler_DeleteFrame(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -753,8 +753,8 @@ func TestHandler_SetIndexTimeQuantum(t *testing.T) { hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { @@ -777,8 +777,8 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { @@ -796,7 +796,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set attributes on the index. @@ -845,7 +845,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set attributes on the index. @@ -895,7 +895,7 @@ func TestHandler_Frame_AddField(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { @@ -999,7 +999,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { @@ -1064,7 +1064,7 @@ func TestHandler_Frame_GetFields(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { @@ -1132,7 +1132,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set bits in the index. @@ -1181,8 +1181,8 @@ func TestHandler_Version(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/version", nil) @@ -1204,9 +1204,9 @@ func TestHandler_Fragment_Nodes(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(3) - h.Cluster.ReplicaN = 2 + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(3) + h.API.Cluster.ReplicaN = 2 w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) @@ -1233,8 +1233,8 @@ func TestHandler_Expvars(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) @@ -1279,8 +1279,8 @@ func TestHandler_CreateInputDefinition(t *testing.T) { ] }`) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { @@ -1314,8 +1314,8 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { defer hldr.Close() hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) //Ensure throwing error if there's duplicated primaryKey field invalidPrimaryKey := []byte(` @@ -1419,8 +1419,8 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) // Test index not found. w := httptest.NewRecorder() @@ -1467,8 +1467,8 @@ func TestHandler_GetInputDefinition(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} @@ -1634,8 +1634,8 @@ func TestHandler_CreateInput(t *testing.T) { "null_value": null }]`) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) // Return error if index does not exist. w := httptest.NewRecorder() @@ -1749,8 +1749,8 @@ func TestInput_JSON(t *testing.T) { err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"}, } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) for _, req := range tests { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(req.json)))) @@ -1810,8 +1810,8 @@ func TestHandler_DeleteView(t *testing.T) { hldr.Index("i0").Frame("f0").SetTimeQuantum("YMD") h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f0/view/standard_2017", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -1836,8 +1836,8 @@ func TestHandler_RecalculateCaches(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) diff --git a/holder_test.go b/holder_test.go index 41ede1adf..4c63e6e76 100644 --- a/holder_test.go +++ b/holder_test.go @@ -393,7 +393,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer hldr1.Close() s := test.NewServer() defer s.Close() - s.Handler.Holder = hldr1.Holder + s.Handler.API.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(client) e.Holder = hldr1.Holder diff --git a/stats_test.go b/stats_test.go index 60deae993..591236268 100644 --- a/stats_test.go +++ b/stats_test.go @@ -215,10 +215,10 @@ func TestStatsCount_CreateIndex(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "createIndex" { t.Errorf("Expected createIndex, Results %s", name) @@ -239,7 +239,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -247,7 +247,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { t.Fatal(err) } called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "deleteIndex" { t.Errorf("Expected deleteIndex, Results %s", name) @@ -268,7 +268,7 @@ func TestStatsCount_CreateFrame(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -276,7 +276,7 @@ func TestStatsCount_CreateFrame(t *testing.T) { t.Fatal(err) } called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { if name != "createFrame" { t.Errorf("Expected createFrame, Results %s", name) @@ -300,7 +300,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() called := false // Create index. @@ -308,7 +308,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { if name != "deleteFrame" { t.Errorf("Expected deleteFrame, Results %s", name) diff --git a/test/handler.go b/test/handler.go index 0f8c1cefd..ae7605603 100644 --- a/test/handler.go +++ b/test/handler.go @@ -27,6 +27,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" ) // Handler represents a test wrapper for pilosa.Handler. @@ -40,10 +41,12 @@ func NewHandler() *Handler { h := &Handler{ Handler: pilosa.NewHandler(), } - h.Handler.Executor = &h.Executor + h.API = pilosa.NewAPI(nil) + h.Handler.API = pilosa.NewAPI(nil) + h.Handler.API.Executor = &h.Executor // Handler test messages can no-op. - h.Broadcaster = pilosa.NopBroadcaster + h.API.Broadcaster = pilosa.NopBroadcaster h.SetNormal() @@ -80,21 +83,35 @@ func NewServer() *Server { if err != nil { panic(err) } + s.Handler.API.URI = uri // Handler test messages can no-op. - s.Handler.Broadcaster = pilosa.NopBroadcaster + s.Handler.API.Broadcaster = pilosa.NopBroadcaster // Create a default cluster on the handler - s.Handler.Cluster = NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri - - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.Cluster = NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() return s } // LocalStatus exists so that test.Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { - return nil, nil + if s.Handler.API.Holder == nil { + return nil, errors.New("Server.Holder is nil") + } + + ns := internal.NodeStatus{ + Host: s.Handler.Handler.API.URI.HostPort(), + State: pilosa.NodeStateUp, + Indexes: pilosa.EncodeIndexes(s.Handler.API.Holder.Indexes()), + } + + // Append Slice list per this Node's indexes + for _, index := range ns.Indexes { + index.Slices = s.Handler.API.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.API.URI.HostPort()) + } + + return &ns, nil } // ClusterStatus exists so that test.Server implements StatusHandler. From be4965d962dcbf216dfb01a9334ca76b878ae66b Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 3 Apr 2018 16:09:11 -0700 Subject: [PATCH 07/79] fixup logger, handler.FileSystem --- api.go | 46 ++++++++++++++++++++++------------------------ handler.go | 5 ++--- server.go | 3 ++- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/api.go b/api.go index 97d7828e5..54dd65244 100644 --- a/api.go +++ b/api.go @@ -19,8 +19,6 @@ import ( "encoding/csv" "fmt" "io" - "io/ioutil" - "log" "net/http" "reflect" "strconv" @@ -44,15 +42,15 @@ type API struct { Cluster *Cluster URI *URI RemoteClient *http.Client - logger *log.Logger + Logger Logger } -func NewAPI(logger *log.Logger) *API { - if logger == nil { - logger = log.New(ioutil.Discard, "", 0) - } +func NewAPI() *API { return &API{ - logger: logger, + Broadcaster: NopBroadcaster, + //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop + //StatusHandler: NopStatusHandler, // TODO: implement the nop + Logger: NopLogger, } } @@ -132,7 +130,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index Meta: options.Encode(), }) if err != nil { - api.logger.Printf("problem sending CreateIndex message: %s", err) + api.Logger.Printf("problem sending CreateIndex message: %s", err) return nil, err } api.Holder.Stats.Count("createIndex", 1, 1.0) @@ -159,7 +157,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { Index: indexName, }) if err != nil { - api.logger.Printf("problem sending DeleteIndex message: %s", err) + api.Logger.Printf("problem sending DeleteIndex message: %s", err) return err } api.Holder.Stats.Count("deleteIndex", 1, 1.0) @@ -187,7 +185,7 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str Meta: options.Encode(), }) if err != nil { - api.logger.Printf("problem sending CreateFrame message: %s", err) + api.Logger.Printf("problem sending CreateFrame message: %s", err) return nil, err } api.Holder.Stats.CountWithCustomTags("createFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) @@ -213,7 +211,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str Frame: frameName, }) if err != nil { - api.logger.Printf("problem sending DeleteFrame message: %s", err) + api.Logger.Printf("problem sending DeleteFrame message: %s", err) return err } api.Holder.Stats.CountWithCustomTags("deleteFrame", 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) @@ -223,7 +221,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { // Validate that this handler owns the slice. if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { - api.logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return ErrClusterDoesNotOwnSlice } @@ -413,7 +411,7 @@ func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inp Definition: def, }) if err != nil { - api.logger.Printf("problem sending CreateInputDefinition message: %s", err) + api.Logger.Printf("problem sending CreateInputDefinition message: %s", err) } return nil } @@ -450,7 +448,7 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp Name: inputDefName, }) if err != nil { - api.logger.Printf("problem sending DeleteInputDefinition message: %s", err) + api.Logger.Printf("problem sending DeleteInputDefinition message: %s", err) } return nil } @@ -521,7 +519,7 @@ func (api *API) CreateFrameField(ctx context.Context, indexName string, frameNam Field: encodeField(field), }) if err != nil { - api.logger.Printf("problem sending CreateField message: %s", err) + api.Logger.Printf("problem sending CreateField message: %s", err) } return err } @@ -546,7 +544,7 @@ func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameNam Field: fieldName, }) if err != nil { - api.logger.Printf("problem sending DeleteField message: %s", err) + api.Logger.Printf("problem sending DeleteField message: %s", err) } return err } @@ -600,7 +598,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri View: viewName, }) if err != nil { - api.logger.Printf("problem sending DeleteView message: %s", err) + api.Logger.Printf("problem sending DeleteView message: %s", err) } return err @@ -685,7 +683,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // Import into fragment. err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) } return err } @@ -699,7 +697,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // Import into fragment. err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) if err != nil { - api.logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) } return err } @@ -750,22 +748,22 @@ func (api *API) ClusterLongQueryTime() time.Duration { func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { // Validate that this handler owns the slice. if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { - api.logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) + api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return nil, nil, ErrClusterDoesNotOwnSlice } // Find the Index. - api.logger.Println("importing:", indexName, frameName, slice) + api.Logger.Printf("importing: %v %v %v", indexName, frameName, slice) index := api.Holder.Index(indexName) if index == nil { - api.logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) + api.Logger.Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrIndexNotFound.Error()) return nil, nil, ErrIndexNotFound } // Retrieve frame. frame := index.Frame(frameName) if frame == nil { - api.logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) + api.Logger.Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", indexName, frameName, slice, ErrFrameNotFound.Error()) return nil, nil, ErrFrameNotFound } return index, frame, nil diff --git a/handler.go b/handler.go index f6a385ccb..6e87bb4be 100644 --- a/handler.go +++ b/handler.go @@ -45,6 +45,8 @@ import ( type Handler struct { Router *mux.Router + FileSystem FileSystem + // The execution engine for running queries. Executor interface { Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) @@ -78,9 +80,6 @@ type errorResponse struct { // NewHandler returns a new instance of Handler with a default logger. func NewHandler() *Handler { handler := &Handler{ - Broadcaster: NopBroadcaster, - //BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop - //StatusHandler: NopStatusHandler, // TODO: implement the nop FileSystem: NopFileSystem, Logger: NopLogger, } diff --git a/server.go b/server.go index 338183cf7..54f68a2f2 100644 --- a/server.go +++ b/server.go @@ -115,13 +115,14 @@ func NewServer() *Server { Logger: NopLogger, } - s.Handler.API = NewAPI(s.logger) + s.Handler.API = NewAPI() s.Handler.API.Holder = s.Holder return s } // Open opens and initializes the server. func (s *Server) Open() error { + s.Handler.API.Logger = s.Logger // TODO do this in NewServer with functional options s.Logger.Printf("open server") // s.ln can be configured prior to Open() via s.OpenListener(). if s.ln == nil { From 17468e7d90bcaa42cd90c6968e1c5a40e805b896 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 5 Apr 2018 19:43:40 +0300 Subject: [PATCH 08/79] Updated tutorials --- docs/tutorials.md | 90 +++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index 2401cdfe4..995e3d41d 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -70,6 +70,8 @@ We now should have `pilosa.local.gossip32` in the current directory with 32 rand Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes. +One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. + Create `node1.config.toml` in the project directory and paste the following in it: ```toml @@ -79,7 +81,7 @@ data-dir = "node1_data" bind = "https://01.pilosa.local:10501" [cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] +coordinator = true [tls] certificate = "pilosa.local.crt" @@ -87,7 +89,7 @@ key = "pilosa.local.key" skip-verify = true [gossip] -seed = "01.pilosa.local:15000" +seeds = ["01.pilosa.local:15000"] port = 15000 key = "pilosa.local.gossip32" ``` @@ -100,16 +102,13 @@ Create `node2.config.toml` in the project directory and paste the following in i data-dir = "node2_data" bind = "https://02.pilosa.local:10502" -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - [tls] certificate = "pilosa.local.crt" key = "pilosa.local.key" skip-verify = true [gossip] -seed = "01.pilosa.local:15000" +seeds = ["01.pilosa.local:15000"] port = 16000 key = "pilosa.local.gossip32" ``` @@ -122,16 +121,13 @@ Create `node3.config.toml` in the project directory and paste the following in i data-dir = "node3_data" bind = "https://03.pilosa.local:10503" -[cluster] -hosts = ["https://01.pilosa.local:10501", "https://02.pilosa.local:10502", "https://03.pilosa.local:10503"] - [tls] certificate = "pilosa.local.crt" key = "pilosa.local.key" skip-verify = true [gossip] -seed = "01.pilosa.local:15000" +seeds = ["01.pilosa.local:15000"] port = 17000 key = "pilosa.local.gossip32" ``` @@ -140,9 +136,9 @@ Here is some explanation of the configuration items: * `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. * `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. -* `[cluster]` section contains the settings for a cluster. `hosts` field is the most important, which contains the list of addresses of other nodes. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings. +* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings. * `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. -* `[gossip]` section contains settings for the Gossip protocol. `seed` is the host and port for the main gossip node which coordinates other nodes. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. +* `[gossip]` section contains settings for the Gossip protocol. `seeds` contain the seed nodes which other nodes gather cluster topology. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. #### Final Touches Before Running the Cluster @@ -189,9 +185,9 @@ curl -k --ipv4 https://01.pilosa.local:10501/status The `-k` flag is used to tell curl that it shouldn't bother with checking the certificate the server provides and `--ipv4` workarounds an issue on MacOS where the curl requests take a long time if the address resolves to `127.0.0.1`. You can leave it out on Linux and WSL. -All nodes should be in the `UP` state: +All nodes should be in the `NORMAL` state: ``` response -{"status":{"Nodes":[{"Host":"01.pilosa.local:10501","State":"UP"},{"Host":"02.pilosa.local:10502","State":"UP"},{"Host":"03.pilosa.local:10503","State":"UP"}]}} +{"state":"NORMAL","nodes":[{"id":"98ebd177-c082-4c54-8d48-7e7c75857b52","uri":{"scheme":"https","host":"02.pilosa.local","port":10502},"isCoordinator":false},{"id":"a33dc0d6-c35f-4559-984a-e582bf032a21","uri":{"scheme":"https","host":"03.pilosa.local","port":10503},"isCoordinator":false},{"id":"e24ac014-ee2f-4cb0-b565-74df6c551f0a","uri":{"scheme":"https","host":"01.pilosa.local","port":10501},"isCoordinator":true}]} ``` #### Running Queries @@ -216,7 +212,7 @@ We just created frame `sample-frame` with default options. Let's run a `SetBit` query: ``` request -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", rowID=1, columnID=100)' +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit(frame="sample-frame", row=1, col=100)' ``` ``` response {"results":[true]} @@ -224,7 +220,7 @@ curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'SetBit Confirm that the bit was indeed set: ``` request -curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)' ``` ``` response {"results":[{"attrs":{},"bits":[100]}]} @@ -232,7 +228,7 @@ curl -k --ipv4 https://01.pilosa.local:10501/index/sample-index/query -d 'Bitmap The same response should be returned when querying other nodes in the cluster: ``` request -curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", rowID=1)' +curl -k --ipv4 https://02.pilosa.local:10502/index/sample-index/query -d 'Bitmap(frame="sample-frame", row=1)' ``` ``` response {"results":[{"attrs":{},"bits":[100]}]} @@ -283,7 +279,7 @@ This query sets the age, weight, and t-cell count for the patient with ID `1` in ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'SetFieldValue(columnID=1, frame="measurements", age=34, weight=128, tcells=1145)' + -d 'SetFieldValue(col=1, frame="measurements", age=34, weight=128, tcells=1145)' ``` ``` response {"results":[null]} @@ -373,11 +369,11 @@ Now, let's add some books to our index. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetColumnAttrs(columnID=1, name="To Kill a Mockingbird", year=1960) - SetColumnAttrs(columnID=2, name="No Name in the Street", year=1972) - SetColumnAttrs(columnID=3, name="The Tipping Point", year=2000) - SetColumnAttrs(columnID=4, name="Out Stealing Horses", year=2003) - SetColumnAttrs(columnID=5, name="The Forever War", year=2008)' + -d 'SetColumnAttrs(col=1, name="To Kill a Mockingbird", year=1960) + SetColumnAttrs(col=2, name="No Name in the Street", year=1972) + SetColumnAttrs(col=3, name="The Tipping Point", year=2000) + SetColumnAttrs(col=4, name="Out Stealing Horses", year=2003) + SetColumnAttrs(col=5, name="The Forever War", year=2008)' ``` ``` response {"results":[null,null,null,null,null]} @@ -387,11 +383,11 @@ And add some members. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetRowAttrs(frame="members", rowID=10001, fullName="John Smith") - SetRowAttrs(frame="members", rowID=10002, fullName="Sue Perkins") - SetRowAttrs(frame="members", rowID=10003, fullName="Jennifer Hawks") - SetRowAttrs(frame="members", rowID=10004, fullName="Pedro Vazquez") - SetRowAttrs(frame="members", rowID=10005, fullName="Pat Washington")' + -d 'SetRowAttrs(frame="members", row=10001, fullName="John Smith") + SetRowAttrs(frame="members", row=10002, fullName="Sue Perkins") + SetRowAttrs(frame="members", row=10003, fullName="Jennifer Hawks") + SetRowAttrs(frame="members", row=10004, fullName="Pedro Vazquez") + SetRowAttrs(frame="members", row=10005, fullName="Pat Washington")' ``` ``` response {"results":[null,null,null,null,null]} @@ -401,7 +397,7 @@ At this point we can query one of the `member` records by querying that row. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", rowID=10002)' + -d 'Bitmap(frame="members", row=10002)' ``` ``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[]}]} @@ -411,23 +407,19 @@ Now let's add some data to the matrix such that each pair represents a member wh ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'SetBit(frame="members", rowID=10001, columnID=3) - SetBit(frame="members", rowID=10001, columnID=5) - - SetBit(frame="members", rowID=10002, columnID=1) - SetBit(frame="members", rowID=10002, columnID=2) - SetBit(frame="members", rowID=10002, columnID=4) - - SetBit(frame="members", rowID=10003, columnID=3) - - SetBit(frame="members", rowID=10004, columnID=4) - SetBit(frame="members", rowID=10004, columnID=5) - - SetBit(frame="members", rowID=10005, columnID=1) - SetBit(frame="members", rowID=10005, columnID=2) - SetBit(frame="members", rowID=10005, columnID=3) - SetBit(frame="members", rowID=10005, columnID=4) - SetBit(frame="members", rowID=10005, columnID=5)' + -d 'SetBit(frame="members", row=10001, col=3) + SetBit(frame="members", row=10001, col=5) + SetBit(frame="members", row=10002, col=1) + SetBit(frame="members", row=10002, col=2) + SetBit(frame="members", row=10002, col=4) + SetBit(frame="members", row=10003, col=3) + SetBit(frame="members", row=10004, col=4) + SetBit(frame="members", row=10004, col=5) + SetBit(frame="members", row=10005, col=1) + SetBit(frame="members", row=10005, col=2) + SetBit(frame="members", row=10005, col=3) + SetBit(frame="members", row=10005, col=4) + SetBit(frame="members", row=10005, col=5)' ``` ``` response {"results":[true,true,true,true,true,true,true,true,true,true,true,true,true]} @@ -437,7 +429,7 @@ Now pull the record for `Sue Perkins` again. ``` request curl localhost:10101/index/books/query \ -X POST \ - -d 'Bitmap(frame="members", rowID=10002)' + -d 'Bitmap(frame="members", row=10002)' ``` ``` response {"results":[{"attrs":{"fullName":"Sue Perkins"},"bits":[1,2,4]}]} @@ -448,7 +440,7 @@ In order to retrieve the attribute information that we stored for each book, we ``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Bitmap(frame="members", rowID=10002)' + -d 'Bitmap(frame="members", row=10002)' ``` ``` response { @@ -466,7 +458,7 @@ Finally, if we want to find out which books were read by both `Sue` and `Pedro`, ``` request curl localhost:10101/index/books/query?columnAttrs=true \ -X POST \ - -d 'Intersect(Bitmap(frame="members", rowID=10002), Bitmap(frame="members", rowID=10004))' + -d 'Intersect(Bitmap(frame="members", row=10002), Bitmap(frame="members", row=10004))' ``` ``` response { From 81cac4775f34ce51ab2d5b2e30e58cd8d59812a9 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 5 Apr 2018 19:53:05 +0300 Subject: [PATCH 09/79] fix cluster config link --- docs/tutorials.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index 995e3d41d..4d1a06c42 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -136,7 +136,7 @@ Here is some explanation of the configuration items: * `data-dir` points to the directory where the Pilosa server writes its data. If it doesn't exist, the server will create it. * `bind` is the address to which the server listens for incoming requests. The address is composed of three parts: scheme, host, and port. The default scheme is `http` so we explicitly specify `https` to use the HTTPS protocol for communication between nodes. -* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-hosts) for other settings. +* `[cluster]` section contains the settings for a cluster. We set `coordinator = true` for only the first node to choose that as the coordinator node. See [Cluster Configuration](../configuration/#cluster-coordinator) for other settings. * `[tls]` section contains the TLS settings, including the path to the SSL certificate and the corresponding key. Set `skip-verify` to `true` in order to disable host name verification and other security measures. Do not set `skip-verify` to `true` on production servers. * `[gossip]` section contains settings for the Gossip protocol. `seeds` contain the seed nodes which other nodes gather cluster topology. There must be at least one gossip seed. The `port` setting is the gossip listen address for the node. It should be different for each node, if the cluster is running on the same computer, otherwise you can set it to the same value. Finally, the `key` points to the gossip encryption key we created before. From cc521e5c91ddee93efb47b56ba11cf8578f9c39a Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 5 Apr 2018 22:29:03 +0300 Subject: [PATCH 10/79] index param is required for /fragment/nodes --- handler.go | 2 +- handler_test.go | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/handler.go b/handler.go index ffd28da46..02d20564c 100644 --- a/handler.go +++ b/handler.go @@ -122,7 +122,7 @@ func BuildRouters(handler *Handler) { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = QueryValidationSpecRequired("slice").Optional("index") + h.validators["GetFragmentNodes"] = QueryValidationSpecRequired("slice", "index") h.validators["GetSliceMax"] = QueryValidationSpecRequired().Optional("inverse") h.validators["PostQuery"] = QueryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") h.validators["GetExport"] = QueryValidationSpecRequired("index", "frame", "view", "slice") diff --git a/handler_test.go b/handler_test.go index ba9480544..12c145e87 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1225,6 +1225,13 @@ func TestHandler_Fragment_Nodes(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } + // index is required + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil) + h.ServeHTTP(w, r) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } } // Ensure the handler can return expvars without panicking. From 6d3f079b7aa73c8c8c57a49e8384e9fa3effb26e Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 6 Apr 2018 17:34:46 +0300 Subject: [PATCH 11/79] updated row/col labels in getting started --- docs/getting-started.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index dc1e07ceb..c9cdb8aba 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -129,7 +129,7 @@ Which repositories did user 14 star: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'Bitmap(frame="stargazer", rowID=14)' + -d 'Bitmap(frame="stargazer", row=14)' ``` ``` response { @@ -167,8 +167,8 @@ Which repositories were starred by user 14 and 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(frame="stargazer", rowID=14), - Bitmap(frame="stargazer", rowID=19) + Bitmap(frame="stargazer", row=14), + Bitmap(frame="stargazer", row=19) )' ``` ``` response @@ -187,8 +187,8 @@ Which repositories were starred by user 14 or 19: curl localhost:10101/index/repository/query \ -X POST \ -d 'Union( - Bitmap(frame="stargazer", rowID=14), - Bitmap(frame="stargazer", rowID=19) + Bitmap(frame="stargazer", row=14), + Bitmap(frame="stargazer", row=19) )' ``` ``` response @@ -207,9 +207,9 @@ Which repositories were starred by user 14 and 19 and also were written in langu curl localhost:10101/index/repository/query \ -X POST \ -d 'Intersect( - Bitmap(frame="stargazer", rowID=14), - Bitmap(frame="stargazer", rowID=19), - Bitmap(frame="language", rowID=1) + Bitmap(frame="stargazer", row=14), + Bitmap(frame="stargazer", row=19), + Bitmap(frame="language", row=1) )' ``` ``` response @@ -227,7 +227,7 @@ Set user 99999 as a stargazer for repository 77777: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", columnID=77777, rowID=99999)' + -d 'SetBit(frame="stargazer", column=77777, row=99999)' ``` ``` response {"results":[true]} From f8cb579187ab6730a6b31b2c588ce993b7431565 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 9 Apr 2018 14:29:43 -0500 Subject: [PATCH 12/79] get tests passing --- api.go | 79 ++++++++++-- client_test.go | 2 +- cluster.go | 6 +- ctl/backup_test.go | 2 +- ctl/export_test.go | 2 +- ctl/import_test.go | 4 +- ctl/restore_test.go | 2 +- handler.go | 272 ++++++++++++++++++----------------------- handler_test.go | 11 +- pilosa.go | 4 + server/cluster_test.go | 14 +-- server/server.go | 1 + server/server_test.go | 6 +- test/handler.go | 24 +--- 14 files changed, 222 insertions(+), 207 deletions(-) diff --git a/api.go b/api.go index 54dd65244..82ab3a4ce 100644 --- a/api.go +++ b/api.go @@ -28,6 +28,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" ) type API struct { @@ -40,7 +41,7 @@ type API struct { BroadcastHandler BroadcastHandler StatusHandler StatusHandler Cluster *Cluster - URI *URI + URI URI RemoteClient *http.Client Logger Logger } @@ -220,7 +221,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { // Validate that this handler owns the slice. - if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return ErrClusterDoesNotOwnSlice } @@ -339,7 +340,7 @@ func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName st // Loop over each slice and import it if this node owns it. for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { // Ignore this slice if we don't own it. - if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { continue } @@ -475,8 +476,13 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s return nil } -func (api *API) RecalculateCaches(ctx context.Context) { +func (api *API) RecalculateCaches(ctx context.Context) error { + err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) + if err != nil { + return errors.Wrap(err, "broacasting message") + } api.Holder.RecalculateCaches() + return nil } func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error { @@ -487,7 +493,7 @@ func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error return nil } -func (api *API) LocalID(ctx context.Context) string { +func (api *API) LocalID() string { return api.Cluster.Node.ID } @@ -549,7 +555,7 @@ func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameNam return err } -func (api *API) FrameFields(ctx context.Context, indexName string, frameName string) (*FrameSchema, error) { +func (api *API) FrameFields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { index := api.Holder.index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -724,13 +730,14 @@ func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, fr return frame.SetTimeQuantum(timeQuantum) } -func (api *API) SliceMax(ctx context.Context, inverse bool) map[string]uint64 { - if inverse { - return api.Holder.MaxInverseSlices() - } +func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxSlices() } +func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 { + return api.Holder.MaxInverseSlices() +} + func (api *API) StatsWithTags(tags []string) StatsClient { if api.Holder == nil || api.Cluster == nil { return nil @@ -747,7 +754,7 @@ func (api *API) ClusterLongQueryTime() time.Duration { func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { // Validate that this handler owns the slice. - if !api.Cluster.OwnsFragment(api.URI.HostPort(), indexName, slice) { + if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return nil, nil, ErrClusterDoesNotOwnSlice } @@ -837,3 +844,53 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na } return setBits, nil } + +func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { + oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) + newNode = api.Cluster.nodeByID(id) + if newNode == nil { + return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") + } + + // If the new coordinator is this node, do the SetCoordinator directly. + if newNode.ID == api.LocalID() { + return oldNode, newNode, api.Cluster.SetCoordinator(newNode) + } + + // Send the set-coordinator message to new node. + err = api.Broadcaster.SendTo( + newNode, + &internal.SetCoordinatorMessage{ + New: EncodeNode(newNode), + }) + if err != nil { + return nil, nil, fmt.Errorf("problem sending SetCoordinator message: %s", err) + } + return oldNode, newNode, nil +} + +func (api *API) RemoveNode(id string) (*Node, error) { + removeNode := api.Cluster.nodeByID(id) + if removeNode == nil { + return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") + } + + // Start the resize process (similar to NodeJoin) + err := api.Cluster.NodeLeave(removeNode) + if err != nil { + return removeNode, errors.Wrap(err, "calling node leave") + } + return removeNode, nil +} + +func (api *API) ResizeAbort() error { + if !api.Cluster.IsCoordinator() { + return ErrNodeNotCoordinator + } + err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted) + return errors.Wrap(err, "complete current job") +} + +func (api *API) State() string { + return api.Cluster.State() +} diff --git a/client_test.go b/client_test.go index 36b9dc38d..cdbaba788 100644 --- a/client_test.go +++ b/client_test.go @@ -86,7 +86,7 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} for i, num := range sliceNums { - owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].Host()) + owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].HostURI()) ownsNum := false for _, ownNum := range owns { if ownNum == num { diff --git a/cluster.go b/cluster.go index 7e53628ac..c490c58cb 100644 --- a/cluster.go +++ b/cluster.go @@ -1198,7 +1198,7 @@ func (c *Cluster) CompleteCurrentJob(state string) error { c.mu.Lock() defer c.mu.Unlock() if c.currentJob == nil { - return fmt.Errorf("no resize job currently running") + return ErrResizeNotRunning } c.currentJob.SetState(state) c.currentJob = nil @@ -1747,7 +1747,7 @@ func (c *Cluster) nodeJoin(node *Node) error { func (c *Cluster) NodeLeave(node *Node) error { // Refuse the request if this is not the coordinator. if !c.IsCoordinator() { - return fmt.Errorf("Node removal requests are only valid on the Coordinator node: %s", c.CoordinatorNode().ID) + return fmt.Errorf("node removal requests are only valid on the coordinator node: %s", c.CoordinatorNode().ID) } if c.State() != ClusterStateNormal { @@ -1761,7 +1761,7 @@ func (c *Cluster) NodeLeave(node *Node) error { // Prevent removing the coordinator node (this node). if node.ID == c.Node.ID { - return fmt.Errorf("The coordinator node cannot be removed. First, make a different node the new coordinator.") + return fmt.Errorf("coordinator cannot be removed; first, make a different node the new coordinator.") } // See if resize job can be generated diff --git a/ctl/backup_test.go b/ctl/backup_test.go index 7a4feb3d9..d80b72475 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -50,7 +50,7 @@ func TestBackupCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/ctl/export_test.go b/ctl/export_test.go index 2bcccd6ba..b414f711f 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -63,7 +63,7 @@ func TestExportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/ctl/import_test.go b/ctl/import_test.go index 119a56071..bd259e342 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -69,7 +69,7 @@ func TestImportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder @@ -110,7 +110,7 @@ func TestImportCommand_RunValue(t *testing.T) { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/ctl/restore_test.go b/ctl/restore_test.go index f1edb7fe9..04cb30415 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -52,7 +52,7 @@ func TestRestoreCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri s.Handler.API.Cluster = test.NewCluster(1) s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() s.Handler.API.Holder = hldr.Holder diff --git a/handler.go b/handler.go index 6e87bb4be..3ac7f6e94 100644 --- a/handler.go +++ b/handler.go @@ -17,7 +17,6 @@ package pilosa import ( "context" "encoding/json" - "errors" "expvar" "fmt" "io" @@ -26,26 +25,27 @@ import ( "net/url" // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" + "reflect" "runtime/debug" "strconv" "strings" "time" - - "reflect" + "unicode" "github.com/gogo/protobuf/proto" "github.com/gorilla/mux" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - - "unicode" + "github.com/pkg/errors" ) // Handler represents an HTTP handler. type Handler struct { Router *mux.Router - FileSystem FileSystem + FileSystem FileSystem + NormalRouter *mux.Router + RestrictedRouter *mux.Router // The execution engine for running queries. Executor interface { @@ -215,7 +215,7 @@ func loadNormal(router *mux.Router, handler *Handler) { } func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) { - http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.Cluster.State()), http.StatusMethodNotAllowed) + http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.API.State()), http.StatusMethodNotAllowed) } func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { @@ -295,7 +295,10 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { return } - cs := pb.(*internal.ClusterStatus) + cs, ok := status.(*internal.ClusterStatus) + if !ok { + panic("status is not a status") + } if err := json.NewEncoder(w).Encode(getStatusResponse{ State: cs.State, Nodes: DecodeNodes(cs.Nodes), @@ -348,28 +351,13 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { - var err error - inverse := false - inverseStr := r.URL.Query().Get("inverse") - if inverseStr != "" { - inverse, err = strconv.ParseBool(r.URL.Query().Get("inverse")) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - } - ms := h.API.SliceMax(r.Context(), inverse) - if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { - pb := &internal.MaxSlicesResponse{ - MaxSlices: ms, - } - if buf, err := proto.Marshal(pb); err != nil { - h.Logger.Printf("protobuf marshal error: %s", err) - } else if _, err := w.Write(buf); err != nil { - h.Logger.Printf("stream write error: %s", err) - } - return +// handleGetSlicesMax handles GET /schema requests. +func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ + Standard: h.API.MaxSlices(r.Context()), + Inverse: h.API.MaxInverseSlices(r.Context()), + }); err != nil { + h.Logger.Printf("write slices-max response error: %s", err) } } @@ -802,7 +790,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - schema, err := h.API.FrameFields(r.Context(), indexName, frameName) + fields, err := h.API.FrameFields(r.Context(), indexName, frameName) if err != nil { switch err { case ErrIndexNotFound: @@ -1557,7 +1545,7 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque } if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { - h.logger().Printf("response encoding error: %s", err) + h.Logger.Printf("response encoding error: %s", err) } } @@ -1646,46 +1634,24 @@ func (h *Handler) handlePostInput(w http.ResponseWriter, r *http.Request) { } } -// <<<<<<< b702f70610962341116d9678ce7c5877b7e171ca -// handlePostClusterResizeSetCoordinator handles POST /cluster/resize/set-coordinator request. func (h *Handler) handlePostClusterResizeSetCoordinator(w http.ResponseWriter, r *http.Request) { // Decode request. var req setCoordinatorRequest err := json.NewDecoder(r.Body).Decode(&req) if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + http.Error(w, "decoding request "+err.Error(), http.StatusBadRequest) return } - oldNode := h.Cluster.nodeByID(h.Cluster.Coordinator) - newNode := h.Cluster.nodeByID(req.ID) - if newNode == nil { - http.Error(w, "Node with provided ID does not exist", http.StatusBadRequest) - return - } - - if err := func() error { - // If the new coordinator is this node, do the SetCoordinator directly. - if newNode.ID == h.Node.ID { - return h.Cluster.SetCoordinator(newNode) + oldNode, newNode, err := h.API.SetCoordinator(r.Context(), req.ID) + if err != nil { + if errors.Cause(err) == ErrNodeIDNotExists { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "setting new coordinator: "+err.Error(), http.StatusInternalServerError) } - - // Send the set-coordinator message to new node. - err := h.Broadcaster.SendTo( - newNode, - &internal.SetCoordinatorMessage{ - New: EncodeNode(newNode), - }) - if err != nil { - return fmt.Errorf("problem sending SetCoordinator message: %s", err) - } - - return nil - }(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) return } - // Encode response. if err := json.NewEncoder(w).Encode(setCoordinatorResponse{ Old: oldNode, @@ -1714,16 +1680,13 @@ func (h *Handler) handlePostClusterResizeRemoveNode(w http.ResponseWriter, r *ht return } - removeNode := h.Cluster.nodeByID(req.ID) - if removeNode == nil { - http.Error(w, fmt.Sprintf("Node is not a member of the cluster: %s", req.ID), http.StatusBadRequest) - return - } - - // Start the resize process (similar to NodeJoin) - err = h.Cluster.NodeLeave(removeNode) + removeNode, err := h.API.RemoveNode(req.ID) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + if errors.Cause(err) == ErrNodeIDNotExists { + http.Error(w, "removing node: "+err.Error(), http.StatusNotFound) + } else { + http.Error(w, "removing node: "+err.Error(), http.StatusInternalServerError) + } return } @@ -1745,21 +1708,20 @@ type removeNodeResponse struct { // handlePostClusterResizeAbort handles POST /cluster/resize/abort request. func (h *Handler) handlePostClusterResizeAbort(w http.ResponseWriter, r *http.Request) { + err := h.API.ResizeAbort() var msg string - - if err := func() error { - if !h.Cluster.IsCoordinator() { - return fmt.Errorf("abort requests must be made on the coordinator node") + if err != nil { + switch errors.Cause(err) { + case ErrNodeNotCoordinator: + http.Error(w, err.Error(), http.StatusBadRequest) + return + case ErrResizeNotRunning: + msg = err.Error() + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + return } - err := h.Cluster.CompleteCurrentJob(ResizeJobStateAborted) - if err != nil { - return err - } - return nil - }(); err != nil { - msg = err.Error() } - // Encode response. if err := json.NewEncoder(w).Encode(clusterResizeAbortResponse{ Info: msg, @@ -1772,80 +1734,82 @@ type clusterResizeAbortResponse struct { Info string `json:"info"` } -// InputJSONDataParser validates input json file and executes SetBit. -func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { - inputDef, err := index.InputDefinition(name) - if err != nil { - return nil, err - } - // If field in input data is not in defined definition, return error. - var colValue uint64 - validFields := make(map[string]bool) - timestampFrame := make(map[string]int64) - for _, field := range inputDef.Fields() { - validFields[field.Name] = true - if field.PrimaryKey { - value, ok := req[field.Name] - if !ok { - return nil, fmt.Errorf("primary key does not exist") - } - rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float - if !ok { - return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) - } - colValue = uint64(rawValue) - } - // Find frame that need to add timestamp. - for _, action := range field.Actions { - if action.ValueDestination == InputSetTimestamp { - timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) - if err != nil { - return nil, err - } - } - } - } +// // InputJSONDataParser validates input json file and executes SetBit. +// func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { +// inputDef, err := index.InputDefinition(name) +// if err != nil { +// return nil, err +// } +// // If field in input data is not in defined definition, return error. +// var colValue uint64 +// validFields := make(map[string]bool) +// timestampFrame := make(map[string]int64) +// for _, field := range inputDef.Fields() { +// validFields[field.Name] = true +// if field.PrimaryKey { +// value, ok := req[field.Name] +// if !ok { +// return nil, fmt.Errorf("primary key does not exist") +// } +// rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float +// if !ok { +// return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) +// } +// colValue = uint64(rawValue) +// } +// // Find frame that need to add timestamp. +// for _, action := range field.Actions { +// if action.ValueDestination == InputSetTimestamp { +// timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) +// if err != nil { +// return nil, err +// } +// } +// } +// } - for key := range req { - _, ok := validFields[key] - if !ok { - return nil, fmt.Errorf("field not found: %s", key) - } - } +// for key := range req { +// _, ok := validFields[key] +// if !ok { +// return nil, fmt.Errorf("field not found: %s", key) +// } +// } - setBits := make(map[string][]*Bit) +// setBits := make(map[string][]*Bit) - for _, field := range inputDef.Fields() { - // skip field that defined in definition but not in input data - if _, ok := req[field.Name]; !ok { - continue - } +// for _, field := range inputDef.Fields() { +// // skip field that defined in definition but not in input data +// if _, ok := req[field.Name]; !ok { +// continue +// } - // Looking into timestampFrame map and set timestamp to the whole frame - for _, action := range field.Actions { - frame := action.Frame - timestamp := timestampFrame[action.Frame] - // Skip input data field values that are set to null - if req[field.Name] == nil { - continue - } - bit, err := HandleAction(action, req[field.Name], colValue, timestamp) - if err != nil { - return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) - } - if bit != nil { - setBits[frame] = append(setBits[frame], bit) - } - } - } - return setBits, nil -} - -// ======= -// >>>>>>> Moved more of handler to API +// // Looking into timestampFrame map and set timestamp to the whole frame +// for _, action := range field.Actions { +// frame := action.Frame +// timestamp := timestampFrame[action.Frame] +// // Skip input data field values that are set to null +// if req[field.Name] == nil { +// continue +// } +// bit, err := HandleAction(action, req[field.Name], colValue, timestamp) +// if err != nil { +// return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) +// } +// if bit != nil { +// setBits[frame] = append(setBits[frame], bit) +// } +// } +// } +// return setBits, nil +// } func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { - h.API.RecalculateCaches(r.Context()) + err := h.API.RecalculateCaches(r.Context()) + if err != nil { + http.Error(w, "recalculating caches: "+err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) } @@ -1901,7 +1865,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques } func (h *Handler) handleGetID(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte(h.API.LocalID(r.Context()))) + _, err := w.Write([]byte(h.API.LocalID())) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } @@ -1936,12 +1900,12 @@ func (s *queryValidationSpec) Optional(args ...string) *queryValidationSpec { func (s queryValidationSpec) validate(query url.Values) error { for _, req := range s.required { if query.Get(req) == "" { - return errors.New(fmt.Sprintf("%s is required", req)) + return errors.Errorf("%s is required", req) } } - for k, _ := range query { + for k := range query { if _, ok := s.args[k]; !ok { - return errors.New(fmt.Sprintf("%s is not a valid argument", k)) + return errors.Errorf("%s is not a valid argument", k) } } return nil diff --git a/handler_test.go b/handler_test.go index 39bcc7fff..c9a3ef988 100644 --- a/handler_test.go +++ b/handler_test.go @@ -158,14 +158,15 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { t.Run("No resize job", func(t *testing.T) { h := test.NewHandler() - h.Cluster = test.NewCluster(1) + h.API.Cluster = test.NewCluster(1) h.SetRestricted() w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"info":"no resize job currently running"}`+"\n" { + bod, err := ioutil.ReadAll(w.Body) + t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err) + } else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" { t.Fatalf("unexpected body: %s", body) } }) @@ -1852,8 +1853,8 @@ func TestHandler_WebUI(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.FileSystem = &statik.FileSystem{} w := httptest.NewRecorder() diff --git a/pilosa.go b/pilosa.go index 04c76a133..0c7e790a3 100644 --- a/pilosa.go +++ b/pilosa.go @@ -78,6 +78,10 @@ var ( ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts") ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") + + ErrNodeIDNotExists = errors.New("node with provided ID does not exist") + ErrNodeNotCoordinator = errors.New("node is not the coordinator") + ErrResizeNotRunning = errors.New("no resize job currently running") ) // Regular expression to validate index and frame names. diff --git a/server/cluster_test.go b/server/cluster_test.go index 6bbdea4b4..99cd62225 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -62,7 +62,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m0.Server.Cluster.MemberSet = gossipMemberSet0 m0.Server.Broadcaster = m0.Server m0.Server.Gossiper = gossipMemberSet0 - m0.Server.Handler.Broadcaster = m0.Server.Broadcaster + m0.Server.Handler.API.Broadcaster = m0.Server.Broadcaster m0.Server.Holder.Broadcaster = m0.Server.Broadcaster m0.Server.BroadcastReceiver = gossipMemberSet0 @@ -89,7 +89,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m1.Server.Cluster.MemberSet = gossipMemberSet1 m1.Server.Broadcaster = m1.Server m1.Server.Gossiper = gossipMemberSet1 - m1.Server.Handler.Broadcaster = m1.Server.Broadcaster + m1.Server.Handler.API.Broadcaster = m1.Server.Broadcaster m1.Server.Holder.Broadcaster = m1.Server.Broadcaster m1.Server.BroadcastReceiver = gossipMemberSet1 @@ -507,9 +507,9 @@ func TestClusterResize_RemoveNode(t *testing.T) { t.Run("ErrorRemoveInvalidNode", func(t *testing.T) { resp := test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), `{"id": "invalid-node-id"}`) - expBody := "Node is not a member of the cluster: invalid-node-id" - if resp.StatusCode != http.StatusBadRequest { - t.Fatalf("expected StatusCode %d but got %d", http.StatusBadRequest, resp.StatusCode) + expBody := "removing node: finding node to remove: node with provided ID does not exist" + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected StatusCode %d but got %d", http.StatusNotFound, resp.StatusCode) } else if strings.TrimSpace(resp.Body) != expBody { t.Fatalf("expected Body '%s' but got '%s'", expBody, strings.TrimSpace(resp.Body)) } @@ -521,7 +521,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { resp = test.MustDo("POST", m0.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) - expBody := "The coordinator node cannot be removed. First, make a different node the new coordinator." + expBody := "removing node: calling node leave: coordinator cannot be removed; first, make a different node the new coordinator." if resp.StatusCode != http.StatusInternalServerError { t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) } else if strings.TrimSpace(resp.Body) != expBody { @@ -538,7 +538,7 @@ func TestClusterResize_RemoveNode(t *testing.T) { resp = test.MustDo("POST", m1.URL()+fmt.Sprintf("/cluster/resize/remove-node"), fmt.Sprintf(`{"id": "%s"}`, nodeID)) - expBody := fmt.Sprintf("Node removal requests are only valid on the Coordinator node: %s", coordinatorNodeID) + expBody := fmt.Sprintf("removing node: calling node leave: node removal requests are only valid on the coordinator node: %s", coordinatorNodeID) if resp.StatusCode != http.StatusInternalServerError { t.Fatalf("expected StatusCode %d but got %d", http.StatusInternalServerError, resp.StatusCode) } else if strings.TrimSpace(resp.Body) != expBody { diff --git a/server/server.go b/server/server.go index a50ad3e71..62c7a70a5 100644 --- a/server/server.go +++ b/server/server.go @@ -219,6 +219,7 @@ func (m *Command) SetupServer() error { c := pilosa.GetHTTPClient(TLSConfig) m.Server.RemoteClient = c m.Server.Handler.API.RemoteClient = c + m.Server.Cluster.RemoteClient = c // Statik file system. m.Server.Handler.FileSystem = &statik.FileSystem{} diff --git a/server/server_test.go b/server/server_test.go index 3852845a6..9f6cdfa8a 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -409,7 +409,11 @@ func TestMain_RecalculateHashes(t *testing.T) { } // Calculate caches on the first node - cluster[0].RecalculateCaches() + err := cluster[0].RecalculateCaches() + if err != nil { + t.Fatalf("recalculating caches: %v", err) + } + target := `{"results":[[{"id":7,"count":99},{"id":1,"count":99},{"id":9,"count":99},{"id":5,"count":99},{"id":4,"count":99},{"id":8,"count":99},{"id":2,"count":99},{"id":6,"count":99},{"id":3,"count":99}]]}` // Run a TopN query on all nodes. The result should be the same as the target. diff --git a/test/handler.go b/test/handler.go index ae7605603..167187e0b 100644 --- a/test/handler.go +++ b/test/handler.go @@ -27,7 +27,6 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" - "github.com/pkg/errors" ) // Handler represents a test wrapper for pilosa.Handler. @@ -41,8 +40,8 @@ func NewHandler() *Handler { h := &Handler{ Handler: pilosa.NewHandler(), } - h.API = pilosa.NewAPI(nil) - h.Handler.API = pilosa.NewAPI(nil) + h.API = pilosa.NewAPI() + h.Handler.API = pilosa.NewAPI() h.Handler.API.Executor = &h.Executor // Handler test messages can no-op. @@ -83,7 +82,7 @@ func NewServer() *Server { if err != nil { panic(err) } - s.Handler.API.URI = uri + s.Handler.API.URI = *uri // Handler test messages can no-op. s.Handler.API.Broadcaster = pilosa.NopBroadcaster @@ -96,22 +95,7 @@ func NewServer() *Server { // LocalStatus exists so that test.Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { - if s.Handler.API.Holder == nil { - return nil, errors.New("Server.Holder is nil") - } - - ns := internal.NodeStatus{ - Host: s.Handler.Handler.API.URI.HostPort(), - State: pilosa.NodeStateUp, - Indexes: pilosa.EncodeIndexes(s.Handler.API.Holder.Indexes()), - } - - // Append Slice list per this Node's indexes - for _, index := range ns.Indexes { - index.Slices = s.Handler.API.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.API.URI.HostPort()) - } - - return &ns, nil + return nil, nil } // ClusterStatus exists so that test.Server implements StatusHandler. From 96e953e6e07f5c894ece6d4b8b93920552a1efe2 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 9 Apr 2018 16:42:50 -0500 Subject: [PATCH 13/79] rename var to be more descriptive --- handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handler.go b/handler.go index 3ac7f6e94..bb9fbc441 100644 --- a/handler.go +++ b/handler.go @@ -289,13 +289,13 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - status, err := h.API.Status(r.Context()) + pb, err := h.API.Status(r.Context()) if err != nil { h.Logger.Printf("cluster status error: %s", err) return } - cs, ok := status.(*internal.ClusterStatus) + cs, ok := pb.(*internal.ClusterStatus) if !ok { panic("status is not a status") } From 401c9682b8bfd0e6983c8d59b4d9e6cf7b17dcf9 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 10 Apr 2018 09:40:28 -0500 Subject: [PATCH 14/79] remove commented code and fix double-NewAPI --- handler.go | 69 ------------------------------------------------- test/handler.go | 2 +- 2 files changed, 1 insertion(+), 70 deletions(-) diff --git a/handler.go b/handler.go index bb9fbc441..d7536f738 100644 --- a/handler.go +++ b/handler.go @@ -1734,75 +1734,6 @@ type clusterResizeAbortResponse struct { Info string `json:"info"` } -// // InputJSONDataParser validates input json file and executes SetBit. -// func (h *Handler) InputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { -// inputDef, err := index.InputDefinition(name) -// if err != nil { -// return nil, err -// } -// // If field in input data is not in defined definition, return error. -// var colValue uint64 -// validFields := make(map[string]bool) -// timestampFrame := make(map[string]int64) -// for _, field := range inputDef.Fields() { -// validFields[field.Name] = true -// if field.PrimaryKey { -// value, ok := req[field.Name] -// if !ok { -// return nil, fmt.Errorf("primary key does not exist") -// } -// rawValue, ok := value.(float64) // The default JSON marshalling will interpret this as a float -// if !ok { -// return nil, fmt.Errorf("float64 require, got value:%s, type: %s", value, reflect.TypeOf(value)) -// } -// colValue = uint64(rawValue) -// } -// // Find frame that need to add timestamp. -// for _, action := range field.Actions { -// if action.ValueDestination == InputSetTimestamp { -// timestampFrame[action.Frame], err = GetTimeStamp(req, field.Name) -// if err != nil { -// return nil, err -// } -// } -// } -// } - -// for key := range req { -// _, ok := validFields[key] -// if !ok { -// return nil, fmt.Errorf("field not found: %s", key) -// } -// } - -// setBits := make(map[string][]*Bit) - -// for _, field := range inputDef.Fields() { -// // skip field that defined in definition but not in input data -// if _, ok := req[field.Name]; !ok { -// continue -// } - -// // Looking into timestampFrame map and set timestamp to the whole frame -// for _, action := range field.Actions { -// frame := action.Frame -// timestamp := timestampFrame[action.Frame] -// // Skip input data field values that are set to null -// if req[field.Name] == nil { -// continue -// } -// bit, err := HandleAction(action, req[field.Name], colValue, timestamp) -// if err != nil { -// return nil, fmt.Errorf("error handling action: %s, err: %s", action.ValueDestination, err) -// } -// if bit != nil { -// setBits[frame] = append(setBits[frame], bit) -// } -// } -// } -// return setBits, nil -// } - func (h *Handler) handleRecalculateCaches(w http.ResponseWriter, r *http.Request) { err := h.API.RecalculateCaches(r.Context()) if err != nil { diff --git a/test/handler.go b/test/handler.go index 167187e0b..cb90cd383 100644 --- a/test/handler.go +++ b/test/handler.go @@ -41,7 +41,7 @@ func NewHandler() *Handler { Handler: pilosa.NewHandler(), } h.API = pilosa.NewAPI() - h.Handler.API = pilosa.NewAPI() + h.Handler.API = h.API h.Handler.API.Executor = &h.Executor // Handler test messages can no-op. From 3d595e6966f8a8643b36120e52d58b65dac08d22 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 10 Apr 2018 15:36:18 -0500 Subject: [PATCH 15/79] make sure that slices is treated as inverseSlices on inverse calls --- executor.go | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/executor.go b/executor.go index 2154b60ff..e74cd63ba 100644 --- a/executor.go +++ b/executor.go @@ -84,29 +84,31 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // to send queries to different slices based on orientation. var inverseSlices []uint64 - // If slices aren't specified, then include all of them. - if len(slices) == 0 { - // Determine slices and inverseSlices for use in e.executeCall(). - if needsSlices { - // Round up the number of slices. - idx := e.Holder.Index(index) - if idx == nil { - return nil, ErrIndexNotFound - } - maxSlice := idx.MaxSlice() - maxInverseSlice := idx.MaxInverseSlice() + // If slices are specified, then use that value for slices or + // inverseSlices. If slices aren't specified, then include all of them. + if len(slices) > 0 { + // For inverse queries, the values of `slices` provided to the Execute() method + // on the remote node actually represents inverseSlices. + inverseSlices = slices + } else if needsSlices { + // Round up the number of slices. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + maxSlice := idx.MaxSlice() + maxInverseSlice := idx.MaxInverseSlice() - // Generate a slices of all slices. - slices = make([]uint64, maxSlice+1) - for i := range slices { - slices[i] = uint64(i) - } + // Generate a slices of all slices. + slices = make([]uint64, maxSlice+1) + for i := range slices { + slices[i] = uint64(i) + } - // Generate a slices of all inverse slices. - inverseSlices = make([]uint64, maxInverseSlice+1) - for i := range inverseSlices { - inverseSlices[i] = uint64(i) - } + // Generate a slices of all inverse slices. + inverseSlices = make([]uint64, maxInverseSlice+1) + for i := range inverseSlices { + inverseSlices[i] = uint64(i) } } @@ -118,7 +120,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { - if call.SupportsInverse() && needsSlices { // Fetch frame & row label based on argument. frame, _ := call.Args["frame"].(string) @@ -147,7 +148,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // executeCall executes a call. func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { - if err := e.validateCallArgs(c); err != nil { return nil, err } From 07ca6d57ad19f87412211efe760d81e1ed66f21a Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 11 Apr 2018 09:42:05 -0500 Subject: [PATCH 16/79] variety of cleanup in api and handler don't export QueryValidationSpecRequired add docs to some methods simplify /debug/vars handling using expvar.Handler() implement Version in API and simplify --- api.go | 22 +++++++++++++++++++--- handler.go | 52 ++++++++++++++++------------------------------------ 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/api.go b/api.go index 82ab3a4ce..35d478d30 100644 --- a/api.go +++ b/api.go @@ -31,6 +31,8 @@ import ( "github.com/pkg/errors" ) +// API provides the top level programmatic interface to Pilosa. It is usually +// wrapped by a handler which provides an external interface (e.g. HTTP). type API struct { Holder *Holder // The execution engine for running queries. @@ -46,6 +48,7 @@ type API struct { Logger Logger } +// NewAPI returns a new API instance. func NewAPI() *API { return &API{ Broadcaster: NopBroadcaster, @@ -55,7 +58,8 @@ func NewAPI() *API { } } -func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryResponse, error) { +// ExecuteQuery parses a PQL query out of the request and executes it. +func (api *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryResponse, error) { resp := QueryResponse{} q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() @@ -67,7 +71,7 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons ExcludeAttrs: req.ExcludeAttrs, ExcludeBits: req.ExcludeBits, } - results, err := a.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) + results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) if err != nil { return resp, err } @@ -86,7 +90,7 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons } // Retrieve column attributes across all calls. - columnAttrSets, err := a.readColumnAttrSets(a.Holder.Index(req.Index), columnIDs) + columnAttrSets, err := api.readColumnAttrSets(api.Holder.Index(req.Index), columnIDs) if err != nil { return resp, err } @@ -118,6 +122,7 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet return ax, nil } +// CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { // Create index. index, err := api.Holder.CreateIndex(indexName, options) @@ -845,6 +850,7 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na return setBits, nil } +// SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) newNode = api.Cluster.nodeByID(id) @@ -869,6 +875,8 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode return oldNode, newNode, nil } +// RemoveNode puts the cluster into the "RESIZING" state and begins the job of +// removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { removeNode := api.Cluster.nodeByID(id) if removeNode == nil { @@ -891,6 +899,14 @@ func (api *API) ResizeAbort() error { return errors.Wrap(err, "complete current job") } +// State returns the cluster state which is usually "NORMAL", but could be +// "STARTING", "RESIZING", or potentially others. See cluster.go for more +// details. func (api *API) State() string { return api.Cluster.State() } + +// Version returns the Pilosa version. +func (api *API) Version() string { + return strings.TrimPrefix(Version, "v") +} diff --git a/handler.go b/handler.go index d7536f738..4cedeb866 100644 --- a/handler.go +++ b/handler.go @@ -108,14 +108,14 @@ func BuildRouters(handler *Handler) { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} - h.validators["GetFragmentNodes"] = QueryValidationSpecRequired("slice").Optional("index") - h.validators["GetSliceMax"] = QueryValidationSpecRequired().Optional("inverse") - h.validators["PostQuery"] = QueryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") - h.validators["GetExport"] = QueryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["GetFragmentData"] = QueryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["PostFragmentData"] = QueryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["GetFragmentBlocks"] = QueryValidationSpecRequired("index", "frame", "view", "slice") - h.validators["PostFrameRestore"] = QueryValidationSpecRequired("host") + h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice").Optional("index") + h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse") + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") + h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "frame", "view", "slice") + h.validators["PostFrameRestore"] = queryValidationSpecRequired("host") } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -154,7 +154,7 @@ func loadCommon(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData") router.HandleFunc("/hosts", handler.handleGetHosts).Methods("GET") router.HandleFunc("/id", handler.handleGetID).Methods("GET") @@ -174,7 +174,7 @@ func loadRestricted(router *mux.Router, handler *Handler) { func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET") + router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") @@ -270,7 +270,7 @@ func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { } filesystem, err := h.FileSystem.New() if err != nil { - h.writeQueryResponse(w, r, &QueryResponse{Err: err}) + _ = h.writeQueryResponse(w, r, &QueryResponse{Err: err}) h.Logger.Printf("Pilosa WebUI is not available. Please run `make generate-statik` before building Pilosa with `make install`.") return } @@ -1337,36 +1337,16 @@ func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { // handleGetVersion handles /version requests. func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { - version := Version - if strings.HasPrefix(version, "v") { - // make the version string semver-compatible - version = version[1:] - } - if err := json.NewEncoder(w).Encode(struct { + err := json.NewEncoder(w).Encode(struct { Version string `json:"version"` }{ - Version: version, - }); err != nil { + Version: h.API.Version(), + }) + if err != nil { h.Logger.Printf("write version response error: %s", err) } } -// handleExpvar handles /debug/vars requests. -func (h *Handler) handleExpvar(w http.ResponseWriter, r *http.Request) { - // Copied from $GOROOT/src/expvar/expvar.go - w.Header().Set("Content-Type", "application/json; charset=utf-8") - fmt.Fprintf(w, "{\n") - first := true - expvar.Do(func(kv expvar.KeyValue) { - if !first { - fmt.Fprintf(w, ",\n") - } - first = false - fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value) - }) - fmt.Fprintf(w, "\n}\n") -} - // QueryResult types. const ( QueryResultTypeNil uint32 = iota @@ -1809,7 +1789,7 @@ type queryValidationSpec struct { args map[string]struct{} } -func QueryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { +func queryValidationSpecRequired(requiredArgs ...string) *queryValidationSpec { args := map[string]struct{}{} for _, arg := range requiredArgs { args[arg] = struct{}{} From 0d27139c04b4561ea9871d7ab4e38ade3ea4468e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 11 Apr 2018 12:51:26 -0500 Subject: [PATCH 17/79] test the inverseSlice fix --- server/server_test.go | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/server/server_test.go b/server/server_test.go index da62273a4..497ffd7da 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -26,6 +26,7 @@ import ( "strings" "testing" "testing/quick" + "time" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" @@ -234,6 +235,49 @@ func TestMain_SetColumnAttrs(t *testing.T) { } } +// Ensure inverse slices get handled correctly in a multi-node query. +func TestMain_InverseSlices(t *testing.T) { + mains := test.MustRunMainWithCluster(t, 2) + + m0 := mains[0] + m1 := mains[1] + + // Make sure to use node0 in the cluster. + var m *test.Main + if m0.Server.NodeID < m1.Server.NodeID { + m = m0 + } else { + m = m1 + } + + // Create frames. + client := m.Client() + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal("create index:", err) + } + if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + t.Fatal("create frame:", err) + } + + // Write data on cluster. + if _, err := m.Query("i", "", fmt.Sprintf(` + SetBit(col=1, frame="f", row=1000) + SetBit(col=1, frame="f", row=2000) + SetBit(col=1, frame="f", row=%d) + `, 1*pilosa.SliceWidth)); err != nil { + t.Fatal("setting bits:", err) + } + + time.Sleep(1 * time.Second) + + // Query the cluster. + if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil { + t.Fatal("another bitmap query:", err) + } else if res != fmt.Sprintf(`{"results":[{"attrs":{},"bits":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" { + t.Fatalf("unexpected result: %s", res) + } +} + // Ensure program can set bits on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { mains1 := test.MustRunMainWithCluster(t, 2) From feec5f07e1487b74a76a928f2ef45ae50dca9dd6 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 11 Apr 2018 13:30:35 -0500 Subject: [PATCH 18/79] add more doc comments to api --- api.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/api.go b/api.go index 35d478d30..be20e5143 100644 --- a/api.go +++ b/api.go @@ -151,6 +151,8 @@ func (api *API) ReadIndex(ctx context.Context, indexName string) (*Index, error) return index, nil } +// DeleteIndex removes the named index. If the index is not found it does +// nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // Delete index from the holder. err := api.Holder.DeleteIndex(indexName) @@ -170,6 +172,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { return nil } +// CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { // Find index. index := api.Holder.Index(indexName) @@ -198,6 +201,9 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str return frame, nil } +// DeleteFrame removes the named frame from the named index. If the index is not +// found, an error is returned. If the frame is not found, it is ignored and no +// action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { // Find index. index := api.Holder.Index(indexName) @@ -387,6 +393,8 @@ func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName st return nil } +// ClusterHosts returns a list of the hosts in the cluster including their ID, +// URL, and which is the coordinator. func (api *API) ClusterHosts(ctx context.Context) []*Node { return api.Cluster.Nodes } @@ -481,6 +489,7 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s return nil } +// RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) if err != nil { @@ -498,10 +507,13 @@ func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error return nil } +// LocalID returns the current node's ID. func (api *API) LocalID() string { return api.Cluster.Node.ID } +// Schema returns information about each index in Pilosa including which frames +// and views they contain. func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.Holder.Schema() } @@ -595,7 +607,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // Delete the view. if err := f.DeleteView(viewName); err != nil { - // Ingore this error becuase views do not exist on all nodes due to slice distribution. + // Ignore this error becuase views do not exist on all nodes due to slice distribution. if err != ErrInvalidView { return err } @@ -675,6 +687,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s return attrs, nil } +// Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { @@ -699,6 +712,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { return err } +// ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { @@ -750,6 +764,8 @@ func (api *API) StatsWithTags(tags []string) StatsClient { return api.Holder.Stats.WithTags(tags...) } +// ClusterLongQueryTime returns the configured threshold for logging/statting +// long running queries. func (api *API) ClusterLongQueryTime() time.Duration { if api.Cluster == nil { return 0 From 9daef2180cd84bebdf827c37440b5b4df0814f7e Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 11 Apr 2018 13:49:51 -0500 Subject: [PATCH 19/79] rename a number of API methods --- api.go | 22 +++++++++++----------- handler.go | 16 ++++++++-------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/api.go b/api.go index be20e5143..7c570b59d 100644 --- a/api.go +++ b/api.go @@ -58,8 +58,8 @@ func NewAPI() *API { } } -// ExecuteQuery parses a PQL query out of the request and executes it. -func (api *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryResponse, error) { +// Query parses a PQL query out of the request and executes it. +func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { resp := QueryResponse{} q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() @@ -143,7 +143,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return index, nil } -func (api *API) ReadIndex(ctx context.Context, indexName string) (*Index, error) { +func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { index := api.Holder.Index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -393,9 +393,9 @@ func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName st return nil } -// ClusterHosts returns a list of the hosts in the cluster including their ID, +// Hosts returns a list of the hosts in the cluster including their ID, // URL, and which is the coordinator. -func (api *API) ClusterHosts(ctx context.Context) []*Node { +func (api *API) Hosts(ctx context.Context) []*Node { return api.Cluster.Nodes } @@ -522,7 +522,7 @@ func (api *API) Status(ctx context.Context) (proto.Message, error) { return api.StatusHandler.ClusterStatus() } -func (api *API) CreateFrameField(ctx context.Context, indexName string, frameName string, field *Field) error { +func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -547,7 +547,7 @@ func (api *API) CreateFrameField(ctx context.Context, indexName string, frameNam return err } -func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameName string, fieldName string) error { +func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -572,7 +572,7 @@ func (api *API) DeleteFrameField(ctx context.Context, indexName string, frameNam return err } -func (api *API) FrameFields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { +func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { index := api.Holder.index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -586,7 +586,7 @@ func (api *API) FrameFields(ctx context.Context, indexName string, frameName str return frame.GetFields() } -func (api *API) FrameViews(ctx context.Context, indexName string, frameName string) ([]*View, error) { +func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { // Retrieve views. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -764,9 +764,9 @@ func (api *API) StatsWithTags(tags []string) StatsClient { return api.Holder.Stats.WithTags(tags...) } -// ClusterLongQueryTime returns the configured threshold for logging/statting +// LongQueryTime returns the configured threshold for logging/statting // long running queries. -func (api *API) ClusterLongQueryTime() time.Duration { +func (api *API) LongQueryTime() time.Duration { if api.Cluster == nil { return 0 } diff --git a/handler.go b/handler.go index 4cedeb866..208c5922f 100644 --- a/handler.go +++ b/handler.go @@ -241,7 +241,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Calculate per request StatsD metrics when the handler is fully configured. statsTags := make([]string, 0, 3) - longQueryTime := h.API.ClusterLongQueryTime() + longQueryTime := h.API.LongQueryTime() if longQueryTime > 0 && dif > longQueryTime { h.Logger.Printf("%s %s %v", r.Method, r.URL.String(), dif) statsTags = append(statsTags, "slow_query") @@ -328,7 +328,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // TODO: Remove req.Index = mux.Vars(r)["index"] - resp, err := h.API.ExecuteQuery(r.Context(), req) + resp, err := h.API.Query(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) h.writeQueryResponse(w, r, &QueryResponse{Err: err}) @@ -374,7 +374,7 @@ func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) { // handleGetIndex handles GET /index/ requests. func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] - index, err := h.API.ReadIndex(r.Context(), indexName) + index, err := h.API.Index(r.Context(), indexName) if err != nil { http.Error(w, err.Error(), http.StatusNotFound) return @@ -742,7 +742,7 @@ func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { Max: req.Max, } - if err := h.API.CreateFrameField(r.Context(), indexName, frameName, field); err != nil { + if err := h.API.CreateField(r.Context(), indexName, frameName, field); err != nil { if err == ErrFrameNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { @@ -771,7 +771,7 @@ func (h *Handler) handleDeleteFrameField(w http.ResponseWriter, r *http.Request) frameName := mux.Vars(r)["frame"] fieldName := mux.Vars(r)["field"] - if err := h.API.DeleteFrameField(r.Context(), indexName, frameName, fieldName); err != nil { + if err := h.API.DeleteField(r.Context(), indexName, frameName, fieldName); err != nil { if err == ErrFrameNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { @@ -790,7 +790,7 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - fields, err := h.API.FrameFields(r.Context(), indexName, frameName) + fields, err := h.API.Fields(r.Context(), indexName, frameName) if err != nil { switch err { case ErrIndexNotFound: @@ -824,7 +824,7 @@ func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] frameName := mux.Vars(r)["frame"] - views, err := h.API.FrameViews(r.Context(), indexName, frameName) + views, err := h.API.Views(r.Context(), indexName, frameName) if err != nil { if err == ErrFrameNotFound { http.Error(w, err.Error(), http.StatusNotFound) @@ -1329,7 +1329,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) // handleGetHosts handles /hosts requests. func (h *Handler) handleGetHosts(w http.ResponseWriter, r *http.Request) { - hosts := h.API.ClusterHosts(r.Context()) + hosts := h.API.Hosts(r.Context()) if err := json.NewEncoder(w).Encode(hosts); err != nil { h.Logger.Printf("write version response error: %s", err) } From 9257029da5fb00736d1ac43d5ec89940d0b3e8eb Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 11 Apr 2018 14:25:02 -0500 Subject: [PATCH 20/79] simplify Status endpoint had to update test which was relying on a fake ClusterStatus implementation. Now the status endpoint uses information directly from Cluster.Nodes and Cluster.state - which is what Server (the usual ClusterStatus impl) uses, so it should make no difference for real clusters. --- api.go | 4 ---- handler.go | 17 ++++------------- handler_test.go | 3 ++- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/api.go b/api.go index 7c570b59d..27827fcec 100644 --- a/api.go +++ b/api.go @@ -518,10 +518,6 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.Holder.Schema() } -func (api *API) Status(ctx context.Context) (proto.Message, error) { - return api.StatusHandler.ClusterStatus() -} - func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) diff --git a/handler.go b/handler.go index 208c5922f..4ce8560b5 100644 --- a/handler.go +++ b/handler.go @@ -289,20 +289,11 @@ func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) { // handleGetStatus handles GET /status requests. func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { - pb, err := h.API.Status(r.Context()) - if err != nil { - h.Logger.Printf("cluster status error: %s", err) - return + status := getStatusResponse{ + State: h.API.State(), + Nodes: h.API.Hosts(r.Context()), } - - cs, ok := pb.(*internal.ClusterStatus) - if !ok { - panic("status is not a status") - } - if err := json.NewEncoder(w).Encode(getStatusResponse{ - State: cs.State, - Nodes: DecodeNodes(cs.Nodes), - }); err != nil { + if err := json.NewEncoder(w).Encode(status); err != nil { h.Logger.Printf("write status response error: %s", err) } } diff --git a/handler_test.go b/handler_test.go index b605a3650..c7d2a80b7 100644 --- a/handler_test.go +++ b/handler_test.go @@ -141,6 +141,7 @@ func TestHandler_Status(t *testing.T) { h := test.NewHandler() h.API.Holder = hldr.Holder h.API.Cluster = test.NewCluster(1) + h.API.Cluster.SetState(pilosa.ClusterStateNormal) h.API.StatusHandler = s s.Handler = h @@ -148,7 +149,7 @@ func TestHandler_Status(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"test-node","uri":{"scheme":"http","host":"localhost","port":10101},"isCoordinator":false}]}`+"\n" { + } else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } From 16539bbab12ccb1743714cd59f0e27e63406c228 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 10 Apr 2018 11:16:27 -0500 Subject: [PATCH 21/79] WIP: implement Min/Max BSI queries --- client_test.go | 36 ++++++++- executor.go | 196 ++++++++++++++++++++++++++++++++++++++++++++++- fragment.go | 64 ++++++++++++++++ fragment_test.go | 73 ++++++++++++++++++ frame.go | 40 ++++++++++ view.go | 43 +++++++++++ 6 files changed, 445 insertions(+), 7 deletions(-) diff --git a/client_test.go b/client_test.go index cdbaba788..4db9156c5 100644 --- a/client_test.go +++ b/client_test.go @@ -332,14 +332,44 @@ func TestClient_ImportValue(t *testing.T) { t.Fatal(err) } + // Verify Sum. sum, cnt, err := frame.FieldSum(nil, fld.Name) if err != nil { t.Fatal(err) } - - // Verify data. if sum != 50 || cnt != 3 { - t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt) + t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) + } + + // Verify Min. + min, cnt, err := frame.FieldMin(nil, fld.Name) + if err != nil { + t.Fatal(err) + } + if min != -10 || cnt != 1 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-10, cnt=1", min, cnt) + } + + // Verify Min with Filter. + filter, err := frame.FieldRange(fld.Name, pql.GT, 40) + if err != nil { + t.Fatal(err) + } + min, cnt, err = frame.FieldMin(filter, fld.Name) // TODO: change this to use the client + if err != nil { + t.Fatal(err) + } + if min != -100 || cnt != 0 { + t.Fatalf("unexpected values: got min=%v, count=%v; expected min=-100, cnt=0", min, cnt) + } + + // Verify Max. + max, cnt, err := frame.FieldMax(nil, fld.Name) + if err != nil { + t.Fatal(err) + } + if max != 40 || cnt != 1 { + t.Fatalf("unexpected values: got max=%v, count=%v; expected max=40, cnt=1", max, cnt) } } diff --git a/executor.go b/executor.go index e74cd63ba..022e48dda 100644 --- a/executor.go +++ b/executor.go @@ -157,6 +157,12 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "Sum": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeSum(ctx, index, c, slices, opt) + case "Min": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeFieldMin(ctx, index, c, slices, opt) + case "Max": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeFieldMax(ctx, index, c, slices, opt) case "ClearBit": return e.executeClearBit(ctx, index, c, opt) case "Count": @@ -233,6 +239,76 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl return other, nil } +// executeFieldMin executes a Min() call. +func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (MinCount, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return MinCount{}, errors.New("Min(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return MinCount{}, errors.New("Min(): field required") + } + + if len(c.Children) > 1 { + return MinCount{}, errors.New("Min() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeFieldMinSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(MinCount) + return other.Smaller(v.(MinCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return MinCount{}, err + } + other, _ := result.(MinCount) + + if other.Count == 0 { + return MinCount{}, nil + } + return other, nil +} + +// executeFieldMax executes a Max() call. +func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (MaxCount, error) { + if frame, _ := c.Args["frame"]; frame == "" { + return MaxCount{}, errors.New("Max(): frame required") + } else if field, _ := c.Args["field"]; field == "" { + return MaxCount{}, errors.New("Max(): field required") + } + + if len(c.Children) > 1 { + return MaxCount{}, errors.New("Max() only accepts a single bitmap input") + } + + // Execute calls in bulk on each remote node and merge. + mapFn := func(slice uint64) (interface{}, error) { + return e.executeFieldMaxSlice(ctx, index, c, slice) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(MaxCount) + return other.Larger(v.(MaxCount)) + } + + result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) + if err != nil { + return MaxCount{}, err + } + other, _ := result.(MaxCount) + + if other.Count == 0 { + return MaxCount{}, nil + } + return other, nil +} + // executeBitmapCall executes a call that returns a bitmap. func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) { // Execute calls in bulk on each remote node and merge. @@ -318,7 +394,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * } } -// executeSumCountSlice executes calculates the sum & count for fields on a slice. +// executeSumCountSlice calculates the sum and count for fields on a slice. func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) { var filter *Bitmap if len(c.Children) == 1 { @@ -342,12 +418,12 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq return SumCount{}, nil } - view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) - if view == nil { + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { return SumCount{}, nil } - vsum, vcount, err := view.FieldSum(filter, field.BitDepth()) + vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) if err != nil { return SumCount{}, err } @@ -357,6 +433,84 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq }, nil } +// executeFieldMinSlice calculates the min for fields on a slice. +func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (MinCount, error) { + var filter *Bitmap + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + if err != nil { + return MinCount{}, err + } + filter = bm + } + + frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) + + frame := e.Holder.Frame(index, frameName) + if frame == nil { + return MinCount{}, nil + } + + field := frame.Field(fieldName) + if field == nil { + return MinCount{}, nil + } + + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return MinCount{}, nil + } + + fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth()) + if err != nil { + return MinCount{}, err + } + return MinCount{ + Min: int64(fmin) + field.Min, + Count: int64(fcount), + }, nil +} + +// executeFieldMaxSlice calculates the max for fields on a slice. +func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (MaxCount, error) { + var filter *Bitmap + if len(c.Children) == 1 { + bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) + if err != nil { + return MaxCount{}, err + } + filter = bm + } + + frameName, _ := c.Args["frame"].(string) + fieldName, _ := c.Args["field"].(string) + + frame := e.Holder.Frame(index, frameName) + if frame == nil { + return MaxCount{}, nil + } + + field := frame.Field(fieldName) + if field == nil { + return MaxCount{}, nil + } + + fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) + if fragment == nil { + return MaxCount{}, nil + } + + fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth()) + if err != nil { + return MaxCount{}, err + } + return MaxCount{ + Max: int64(fmax) + field.Min, + Count: int64(fcount), + }, nil +} + // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. @@ -1619,3 +1773,37 @@ func decodeSumCount(pb *internal.SumCount) SumCount { Count: pb.Count, } } + +// MinCount represents a grouping of min and count for Min() calls. +type MinCount struct { + Min int64 `json:"min"` + Count int64 `json:"count"` +} + +// Smaller returns the smaller of the two MinCounts. +func (mc *MinCount) Smaller(other MinCount) MinCount { + if mc.Count == 0 || other.Count < mc.Count { + return other + } + return MinCount{ + Min: mc.Min, + Count: mc.Count, + } +} + +// MaxCount represents a grouping of max and count for Max() calls. +type MaxCount struct { + Max int64 `json:"max"` + Count int64 `json:"count"` +} + +// Larger returns the larger of the two MaxCounts. +func (mc *MaxCount) Larger(other MaxCount) MaxCount { + if mc.Count == 0 || other.Count < mc.Count { + return other + } + return MaxCount{ + Max: mc.Max, + Count: mc.Count, + } +} diff --git a/fragment.go b/fragment.go index 438a80ae7..f40c0c8d9 100644 --- a/fragment.go +++ b/fragment.go @@ -614,6 +614,70 @@ func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, e return sum, count, nil } +// FieldMin returns the min of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. +func (f *Fragment) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) { + + consider := f.Row(uint64(bitDepth)) + if filter != nil { + consider = consider.Intersect(filter) + } + + // If there are no columns to consider, return early. + if consider.Count() == 0 { + return 0, 0, nil + } + + for i := bitDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + row := f.Row(uint64(ii)) + + x := consider.Difference(row) + count = x.Count() + if count > 0 { + consider = x + } else { + min += (1 << ii) + if ii == 0 { + count = consider.Count() + } + } + } + + return min, count, nil +} + +// FieldMax returns the max of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. +func (f *Fragment) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) { + + consider := f.Row(uint64(bitDepth)) + if filter != nil { + consider = consider.Intersect(filter) + } + + // If there are no columns to consider, return early. + if consider.Count() == 0 { + return 0, 0, nil + } + + for i := bitDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + row := f.Row(uint64(ii)) + + x := row.Intersect(consider) + count = x.Count() + if count > 0 { + max += (1 << ii) + consider = x + } else if ii == 0 { + count = consider.Count() + } + } + + return max, count, nil +} + // FieldRange returns bitmaps with a field value encoding matching the predicate. func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { switch op { diff --git a/fragment_test.go b/fragment_test.go index 6a4a2f377..6b49b5d75 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -256,6 +256,79 @@ func TestFragment_FieldSum(t *testing.T) { }) } +// Ensure a fragment can find the max of field values. +func TestFragment_FieldMinMax(t *testing.T) { + const bitDepth = 16 + + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter *pilosa.Bitmap + exp uint64 + cnt uint64 + }{ + {filter: nil, exp: 0, cnt: 1}, + {filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(1), exp: 0, cnt: 0}, + {filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1}, + {filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1}, + } + for i, test := range tests { + if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil { + t.Fatal(err) + } else if min != test.exp { + t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) + } else if cnt != test.cnt { + t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter *pilosa.Bitmap + exp uint64 + cnt uint64 + }{ + {filter: nil, exp: 2818, cnt: 2}, + {filter: pilosa.NewBitmap(2000, 4000, 5000), exp: 2818, cnt: 1}, + {filter: pilosa.NewBitmap(2000, 4000), exp: 300, cnt: 2}, + {filter: pilosa.NewBitmap(1), exp: 0, cnt: 0}, + {filter: pilosa.NewBitmap(1000), exp: 382, cnt: 1}, + {filter: pilosa.NewBitmap(7000), exp: 0, cnt: 1}, + } + for i, test := range tests { + if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil { + t.Fatal(err) + } else if max != test.exp { + t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) + } else if cnt != test.cnt { + t.Errorf("test %d expected cnt: %v, but got: %v", i, test.cnt, cnt) + } + } + }) +} + // Ensure a fragment query for matching fields. func TestFragment_FieldRange(t *testing.T) { const bitDepth = 16 diff --git a/frame.go b/frame.go index def41d1b1..a8213e264 100644 --- a/frame.go +++ b/frame.go @@ -763,6 +763,46 @@ func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err err return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil } +// FieldMin returns the min for a field. +// An optional filtering bitmap can be provided. +func (f *Frame) FieldMin(filter *Bitmap, name string) (min, count int64, err error) { + field := f.Field(name) + if field == nil { + return 0, 0, ErrFieldNotFound + } + + view := f.View(ViewFieldPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmin, vcount, err := view.FieldMin(filter, field.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmin) + field.Min, int64(vcount), nil +} + +// FieldMax returns the max for a field. +// An optional filtering bitmap can be provided. +func (f *Frame) FieldMax(filter *Bitmap, name string) (max, count int64, err error) { + field := f.Field(name) + if field == nil { + return 0, 0, ErrFieldNotFound + } + + view := f.View(ViewFieldPrefix + name) + if view == nil { + return 0, 0, nil + } + + vmax, vcount, err := view.FieldMax(filter, field.BitDepth()) + if err != nil { + return 0, 0, err + } + return int64(vmax) + field.Min, int64(vcount), nil +} + func (f *Frame) FieldRange(name string, op pql.Token, predicate int64) (*Bitmap, error) { // Retrieve and validate field. field := f.Field(name) diff --git a/view.go b/view.go index f775f1b0e..dbcc73b8d 100644 --- a/view.go +++ b/view.go @@ -357,6 +357,49 @@ func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err e return sum, count, nil } +// FieldMin returns the min and count of a field. +func (v *View) FieldMin(filter *Bitmap, bitDepth uint) (min, count uint64, err error) { + var minHasValue bool + for _, f := range v.Fragments() { + fmin, fcount, err := f.FieldMin(filter, bitDepth) + if err != nil { + return min, count, err + } + // Don't consider a min based on zero columns. + if fcount == 0 { + continue + } + + if !minHasValue { + min = fmin + minHasValue = true + count += fcount + continue + } + + if fmin < min { + min = fmin + count += fcount + } + } + return min, count, nil +} + +// FieldMax returns the max and count of a field. +func (v *View) FieldMax(filter *Bitmap, bitDepth uint) (max, count uint64, err error) { + for _, f := range v.Fragments() { + fmax, fcount, err := f.FieldMax(filter, bitDepth) + if err != nil { + return max, count, err + } + if fcount > 0 && fmax > max { + max = fmax + count += fcount + } + } + return max, count, nil +} + // FieldRange returns bitmaps with a field value encoding matching the predicate. func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Bitmap, error) { bm := NewBitmap() From ee4dbbf328735bdff1a38317c985fcb2e8759898 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 10 Apr 2018 18:01:05 -0500 Subject: [PATCH 22/79] min/max documentation --- docs/data-model.md | 2 +- docs/glossary.md | 6 ++++- docs/query-language.md | 50 ++++++++++++++++++++++++++++++++++++++++++ docs/tutorials.md | 46 ++++++++++++++++++++++++++++++++++++-- fragment_test.go | 2 +- 5 files changed, 101 insertions(+), 5 deletions(-) diff --git a/docs/data-model.md b/docs/data-model.md index 6a57a96a1..655d1633e 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -112,7 +112,7 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") #### BSI Range-Encoding Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded -bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers. +bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. diff --git a/docs/glossary.md b/docs/glossary.md index 612c65dd4..6157eba57 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -14,7 +14,7 @@ nav = [] [Bitmap](../data-model/#overview): The on-disk and in-memory representation of a [row](#row). Implemented with [Roaring](#roaring-bitmap). `Bitmap` is also the basic [PQL](#pql) query for reading a Bitmap. -[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi) and [Sum](#sum) queries. +[BSI](../data-model/#bsi-range-encoding) Bit-sliced indexing is the method Pilosa uses to represent multi-bit integers. Integer values are stored in [fields](#field), and can be used for [Range](#range-bsi), [Min](#min), [Max](#max), and [Sum](#sum) queries. Cluster: A cluster consists of one or more [nodes](#node) which share a cluster configuration. The cluster also defines how data is [replicated](#replica) throughout and how internode communication is coordinated. Pilosa does not have a leader node, all data is evenly distributed, and any node can respond to queries. @@ -30,8 +30,12 @@ nav = [] [Jump Consistent Hash](https://arxiv.org/pdf/1406.2294v1.pdf): A fast, minimal memory, consistent hash algorithm that evenly distributes the workload even when the number of buckets changes. +[Max](../query-language/#max): A [PQL](#pql) query that returns the maximum integer value stored in [BSI](#bsi) [fields](#field). + MaxSlice: The total number of [slices](#slice) allocated to handle the current set of [columns](#column). This value is important for all [nodes](#node) to efficiently distribute queries. +[Min](../query-language/#min): A [PQL](#pql) query that returns the minimum integer value stored in [BSI](#bsi) [fields](#field). + Node: An individual running instance of Pilosa server which belongs to a [cluster](#cluster). Partition: The [consistent hash](#jump-consistent-hash) maps keys to partitions (or locations on the unit circle), based on a preset maximum number of partitions. Partitions are then evenly mapped to physical [nodes](#node). To add nodes to the [cluster](#cluster), the partitions must be remapped, and data is then associated across the new cluster topology. `DefaultPartitionN` is 256. It can be modified, but only at compile time, and before ingesting any data. diff --git a/docs/query-language.md b/docs/query-language.md index 6b0584320..efb143996 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -523,6 +523,56 @@ Range(frame="stats", commitactivity >< [100, 200]) This is conceptually equivalent to the interval 100 <= commitactivity <= 200, but this chained comparison syntax is not currently supported. `BETWEEN` query syntax is restricted to greater-than-or-equal-to and less-than-or-equal-to, but any valid interval on the integers can be represented this way. +#### Min + +**Spec:** + +``` +Min([BITMAP_CALL], , ) +``` + +**Description:** + +Returns the minimum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all collumns are considered. + +**Result Type:** object with the min and count of columns containing the min value. + +**Examples:** + +Query the size of all repositories. +``` +Min(frame="stats", field="diskusage") +``` + +Return `{"min":4,"count":2}` + +* Result is the smallest repository in kilobytes, plus the number of repositories of that size. + +#### Max + +**Spec:** + +``` +Max([BITMAP_CALL], , ) +``` + +**Description:** + +Returns the maximum value of all BSI integer values in the `field` in this `frame`. If the optional `Bitmap` call is supplied, only columns with set bits are considered, otherwise all columns are considered. + +**Result Type:** object with the max and count of columns containing the max value. + +**Examples:** + +Query the size of all repositories. +``` +Max(frame="stats", field="diskusage") +``` + +Return `{"max":88,"count":13}` + +* Result is the largest repository in kilobytes, plus the number of repositories of that size. + #### Sum **Spec:** diff --git a/docs/tutorials.md b/docs/tutorials.md index 2401cdfe4..6715e1cd1 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -247,7 +247,7 @@ Check out our [Administration Guide](https://www.pilosa.com/docs/latest/administ #### Introduction -Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range` and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. +Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: ``` request @@ -333,7 +333,7 @@ curl localhost:10101/index/patients/query \ ``` The results you get from the `Sum` query contain the `sum` of all values as well as the `count` of columns with a value. To get the average you can just divide `sum` by `count`. -You can also provide a filter to the `Sum()` function, to find the average age of all patients over 40. +You can also provide a filter to the `Sum()` function to find the average age of all patients over 40. ``` request curl localhost:10101/index/patients/query \ -X POST \ @@ -344,6 +344,48 @@ curl localhost:10101/index/patients/query \ ``` Notice in this case that the count is only `3` because of the `age > 40` filter applied to the query. +To find the minimum age of all patients, run a `Min` query: +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Min(frame="measurements", field="age")' +``` +``` response +{"results":[{"min":19,"count":1}]} +``` +The results you get from the `Min` query contain the `min` of all values as well as the `count` of columns with that value. + +You can also provide a filter to the `Min()` function to find the minimum age of all patients over 40. +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Min(Range(frame="measurements", age > 40), frame="measurements", field="age")' +``` +``` response +{"results":[{"min":57,"count":1}]} +``` + +To find the maximum age of all patients, run a `Max` query: +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Max(frame="measurements", field="age")' +``` +``` response +{"results":[{"max":71,"count":1}]} +``` +The results you get from the `Max` query contain the `max` of all values as well as the `count` of columns with that value. + +You can also provide a filter to the `Max()` function to find the maximum age of all patients under 40. +``` request +curl localhost:10101/index/patients/query \ + -X POST \ + -d 'Max(Range(frame="measurements", age < 40), frame="measurements", field="age")' +``` +``` response +{"results":[{"max":34,"count":1}]} +``` + ### Storing Row and Column Attributes #### Introduction diff --git a/fragment_test.go b/fragment_test.go index 6b49b5d75..d7c97a5cf 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -256,7 +256,7 @@ func TestFragment_FieldSum(t *testing.T) { }) } -// Ensure a fragment can find the max of field values. +// Ensure a fragment can find the min and max of field values. func TestFragment_FieldMinMax(t *testing.T) { const bitDepth = 16 From 04f5d5875c9ab49c48321be08bf0c5d2c1e13195 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 12 Apr 2018 11:28:35 -0500 Subject: [PATCH 23/79] api docs, rename funcs, refactor usage of internal All exported funcs in api.go are now documented Several poorly named methods of API and Cluster were renamed. Particularly, the word Fragment was often changed to Slice in cases where it was really a slice being specified and not a fragment. several methods which received or returned internal data structures have been refactored to be more opaque. Deprecation logging was added to input definition methods. --- api.go | 105 +++++++++++++++++++++++++++++++++++++++++------- cluster.go | 12 +++--- executor.go | 6 +-- fragment.go | 4 +- handler.go | 46 +++++---------------- holder.go | 2 +- pilosa.go | 7 ++++ test/cluster.go | 4 +- 8 files changed, 120 insertions(+), 66 deletions(-) diff --git a/api.go b/api.go index 27827fcec..f57d159f2 100644 --- a/api.go +++ b/api.go @@ -19,6 +19,7 @@ import ( "encoding/csv" "fmt" "io" + "io/ioutil" "net/http" "reflect" "strconv" @@ -143,6 +144,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return index, nil } +// Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { index := api.Holder.Index(indexName) if index == nil { @@ -230,9 +232,11 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str return nil } +// ExportCSV encodes the fragment designated by the index,frame,view,slice as +// CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { // Validate that this handler owns the slice. - if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { + if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return ErrClusterDoesNotOwnSlice } @@ -262,11 +266,20 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin return nil } -func (api *API) FragmentNodes(ctx context.Context, indexName string, slice uint64) []*Node { - return api.Cluster.FragmentNodes(indexName, slice) +// SliceNodes returns the node and all replicas which should contain a slice's data. +func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) []*Node { + return api.Cluster.SliceNodes(indexName, slice) } -func (api *API) FragmentData(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (*Fragment, error) { +// WriterTo is an interface for any Object which knows how to serialize itself to an io.Writer +type WriterTo interface { + WriteTo(w io.Writer) (n int64, err error) +} + +// MarshalFragment returns an object which can write the specified fragment's data +// to an io.Writer. The serialized data can be read back into a fragment with +// the UnmarshalFragment API call. +func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (WriterTo, error) { // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -275,7 +288,10 @@ func (api *API) FragmentData(ctx context.Context, indexName string, frameName st return f, nil } -func (api *API) WriteFragmentData(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { +// UnmarshalFragment creates a new fragment (if necessary) and reads data from a +// Reader which was previously written by MarshalFragment to populate the +// fragment's data. +func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -301,19 +317,38 @@ func (api *API) WriteFragmentData(ctx context.Context, indexName string, frameNa return nil } -func (api *API) FragmentBlockData(ctx context.Context, req internal.BlockDataRequest) (internal.BlockDataResponse, error) { +// FragmentBlockData is an endpoint for internal usage. It is not guaranteed to +// return anything useful. Currently it returns protobuf encoded row and column +// ids from a "block" which is a subdivision of a fragment. +func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { + reqBytes, err := ioutil.ReadAll(body) + if err != nil { + return nil, BadRequestError{errors.Wrap(err, "read body error")} + } + var req internal.BlockDataRequest + if err := proto.Unmarshal(reqBytes, &req); err != nil { + return nil, BadRequestError{errors.Wrap(err, "unmarshal body error")} + } + // Retrieve fragment from holder. f := api.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice) if f == nil { - return internal.BlockDataResponse{}, ErrFragmentNotFound + return nil, ErrFragmentNotFound } - // Read data - var resp internal.BlockDataResponse + var resp = internal.BlockDataResponse{} resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) - return resp, nil + + // Encode response. + buf, err := proto.Marshal(&resp) + if err != nil { + return nil, errors.Wrap(err, "merge block response encoding error: %s") + } + + return buf, nil } +// FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) @@ -326,6 +361,8 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName return blocks, nil } +// RestoreFrame reads all the data that this host should have for a given frame +// from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { // Create a client for the remote cluster. client := NewInternalHTTPClientFromURI(host, api.RemoteClient) @@ -351,7 +388,7 @@ func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName st // Loop over each slice and import it if this node owns it. for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { // Ignore this slice if we don't own it. - if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { + if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { continue } @@ -399,7 +436,10 @@ func (api *API) Hosts(ctx context.Context) []*Node { return api.Cluster.Nodes } +// CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { + api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. +Please open an issue if you need to continue using it.`) // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -430,7 +470,9 @@ func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inp return nil } +// InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { + api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -444,7 +486,9 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN return inputDef, nil } +// DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { + api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -467,7 +511,9 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp return nil } +// WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { + api.Logger.Printf("WriteInput is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -499,10 +545,24 @@ func (api *API) RecalculateCaches(ctx context.Context) error { return nil } -func (api *API) PostClusterMessage(ctx context.Context, pb proto.Message) error { +// PostClusterMessage is for internal use. It decodes a protobuf message out of +// the body and forwards it to the BroadcastHandler. +func (api *API) PostClusterMessage(ctx context.Context, reqBody io.Reader) error { + // Read entire body. + body, err := ioutil.ReadAll(reqBody) + if err != nil { + return errors.Wrap(err, "reading body") + } + + // Marshal into request object. + pb, err := UnmarshalMessage(body) + if err != nil { + return errors.Wrap(err, "unmarshaling message") + } + // Forward the error message. if err := api.BroadcastHandler.ReceiveMessage(pb); err != nil { - return err + return errors.Wrap(err, "receiving message") } return nil } @@ -518,6 +578,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { return api.Holder.Schema() } +// CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) @@ -543,6 +604,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str return err } +// DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) @@ -568,6 +630,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str return err } +// Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { index := api.Holder.index(indexName) if index == nil { @@ -582,6 +645,7 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) return frame.GetFields() } +// Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { // Retrieve views. f := api.Holder.Frame(indexName, frameName) @@ -594,6 +658,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( return views, nil } +// DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { // Retrieve frame. f := api.Holder.Frame(indexName, frameName) @@ -623,6 +688,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri return err } +// IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { // Retrieve index from holder. index := api.Holder.Index(indexName) @@ -723,6 +789,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return err } +// ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { // Retrieve index by name. index := api.Holder.Index(indexName) @@ -734,6 +801,8 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti return index.SetTimeQuantum(timeQuantum) } +// ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO: +// what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { // Retrieve index by name. frame := api.Holder.Frame(indexName, frameName) @@ -745,14 +814,19 @@ func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, fr return frame.SetTimeQuantum(timeQuantum) } +// MaxSlices returns the maximum slice number for each index in a map. func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxSlices() } +// MaxInverseSlices returns the maximum inverse slice number for each index in a +// map. func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxInverseSlices() } +// StatsWithTags returns an instance of whatever implementation of StatsClient +// pilosa is using with the given tags. func (api *API) StatsWithTags(tags []string) StatsClient { if api.Holder == nil || api.Cluster == nil { return nil @@ -771,7 +845,7 @@ func (api *API) LongQueryTime() time.Duration { func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*Index, *Frame, error) { // Validate that this handler owns the slice. - if !api.Cluster.OwnsFragment(api.LocalID(), indexName, slice) { + if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) return nil, nil, ErrClusterDoesNotOwnSlice } @@ -793,7 +867,7 @@ func (api *API) indexFrame(indexName string, frameName string, slice uint64) (*I return index, frame, nil } -// inputJSONDataParser validates input json file and executes SetBit. +// inputJSONDataParser validates input json file and executes SetBit. Deprecated - remove with input definition stuff. func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, name string) (map[string][]*Bit, error) { inputDef, err := index.InputDefinition(name) if err != nil { @@ -903,6 +977,7 @@ func (api *API) RemoveNode(id string) (*Node, error) { return removeNode, nil } +// ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { if !api.Cluster.IsCoordinator() { return ErrNodeNotCoordinator diff --git a/cluster.go b/cluster.go index c490c58cb..abf753892 100644 --- a/cluster.go +++ b/cluster.go @@ -654,7 +654,7 @@ func (c *Cluster) fragsByHost(idx *Index) fragsByHost { func (c *Cluster) fragCombos(idx string, maxSlice uint64, frameViews viewsByFrame) fragsByHost { t := make(fragsByHost) for i := uint64(0); i <= maxSlice; i++ { - nodes := c.FragmentNodes(idx, i) + nodes := c.SliceNodes(idx, i) for _, n := range nodes { // for each frame/view combination: for frame, views := range frameViews { @@ -807,14 +807,14 @@ func (c *Cluster) Partition(index string, slice uint64) int { return int(h.Sum64() % uint64(c.PartitionN)) } -// FragmentNodes returns a list of nodes that own a fragment. -func (c *Cluster) FragmentNodes(index string, slice uint64) []*Node { +// SliceNodes returns a list of nodes that own a fragment. +func (c *Cluster) SliceNodes(index string, slice uint64) []*Node { return c.PartitionNodes(c.Partition(index, slice)) } -// OwnsFragment returns true if a host owns a fragment. -func (c *Cluster) OwnsFragment(nodeID string, index string, slice uint64) bool { - return Nodes(c.FragmentNodes(index, slice)).ContainsID(nodeID) +// OwnsSlice returns true if a host owns a fragment. +func (c *Cluster) OwnsSlice(nodeID string, index string, slice uint64) bool { + return Nodes(c.SliceNodes(index, slice)).ContainsID(nodeID) } // PartitionNodes returns a list of nodes that own a partition. diff --git a/executor.go b/executor.go index 2154b60ff..c2cd860c4 100644 --- a/executor.go +++ b/executor.go @@ -941,7 +941,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) { slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.FragmentNodes(index, slice) { + for _, node := range e.Cluster.SliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.ClearBit(view, rowID, colID, nil) @@ -1042,7 +1042,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.C slice := colID / SliceWidth ret := false - for _, node := range e.Cluster.FragmentNodes(index, slice) { + for _, node := range e.Cluster.SliceNodes(index, slice) { // Update locally if host matches. if node.ID == e.Node.ID { val, err := f.SetBit(view, rowID, colID, timestamp) @@ -1385,7 +1385,7 @@ func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (m loop: for _, slice := range slices { - for _, node := range e.Cluster.FragmentNodes(index, slice) { + for _, node := range e.Cluster.SliceNodes(index, slice) { if Nodes(nodes).Contains(node) { m[node] = append(m[node], slice) continue loop diff --git a/fragment.go b/fragment.go index 438a80ae7..c44a07380 100644 --- a/fragment.go +++ b/fragment.go @@ -1702,7 +1702,7 @@ func (s *FragmentSyncer) isClosing() bool { // then merges any blocks which have differences. func (s *FragmentSyncer) SyncFragment() error { // Determine replica set. - nodes := s.Cluster.FragmentNodes(s.Fragment.Index(), s.Fragment.Slice()) + nodes := s.Cluster.SliceNodes(s.Fragment.Index(), s.Fragment.Slice()) if len(nodes) == 1 { return nil } @@ -1784,7 +1784,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Read pairs from each remote block. var pairSets []PairSet var clients []InternalClient - for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) { + for _, node := range s.Cluster.SliceNodes(f.Index(), f.Slice()) { if s.Node.ID == node.ID { continue } diff --git a/handler.go b/handler.go index 4ce8560b5..33d2e5846 100644 --- a/handler.go +++ b/handler.go @@ -1169,7 +1169,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } // Retrieve fragment owner nodes. - nodes := h.API.FragmentNodes(r.Context(), index, slice) + nodes := h.API.SliceNodes(r.Context(), index, slice) // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { @@ -1188,7 +1188,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) } // Retrieve fragment from holder. - f, err := h.API.FragmentData(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) + f, err := h.API.MarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) if err != nil { http.Error(w, err.Error(), http.StatusNotFound) return @@ -1210,7 +1210,7 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) return } - if err = h.API.WriteFragmentData(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil { + if err = h.API.UnmarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil { if err == ErrFrameNotFound { http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) } else { @@ -1221,19 +1221,11 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) // handleGetFragmentBlockData handles GET /fragment/block/data requests. func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { - // Read request object. - var req internal.BlockDataRequest - if body, err := ioutil.ReadAll(r.Body); err != nil { - http.Error(w, "ready body error", http.StatusBadRequest) - return - } else if err := proto.Unmarshal(body, &req); err != nil { - http.Error(w, "unmarshal body error", http.StatusBadRequest) - return - } - - resp, err := h.API.FragmentBlockData(r.Context(), req) + buf, err := h.API.FragmentBlockData(r.Context(), r.Body) if err != nil { - if err == ErrFragmentNotFound { + if _, ok := err.(BadRequestError); ok { + http.Error(w, err.Error(), http.StatusBadRequest) + } else if err == ErrFragmentNotFound { http.Error(w, err.Error(), http.StatusNotFound) } else { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1241,13 +1233,6 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ return } - // Encode response. - buf, err := proto.Marshal(&resp) - if err != nil { - h.Logger.Printf("merge block response encoding error: %s", err) - return - } - // Write response. w.Header().Set("Content-Type", "application/protobuf") w.Header().Set("Content-Length", strconv.Itoa(len(buf))) @@ -1742,23 +1727,10 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques return } - // Read entire body. - body, err := ioutil.ReadAll(r.Body) + err := h.API.PostClusterMessage(r.Context(), r.Body) if err != nil { + // TODO this was the previous behavior, but perhaps not everything is a bad request http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Marshal into request object. - pb, err := UnmarshalMessage(body) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.API.PostClusterMessage(r.Context(), pb); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return } if err := json.NewEncoder(w).Encode(defaultClusterMessageResponse{}); err != nil { diff --git a/holder.go b/holder.go index 22c53c7a4..570433ce4 100644 --- a/holder.go +++ b/holder.go @@ -614,7 +614,7 @@ func (s *HolderSyncer) SyncHolder() error { for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. - if !s.Cluster.OwnsFragment(s.Node.ID, di.Name, slice) { + if !s.Cluster.OwnsSlice(s.Node.ID, di.Name, slice) { continue } diff --git a/pilosa.go b/pilosa.go index 4b28daccb..ffb836ae8 100644 --- a/pilosa.go +++ b/pilosa.go @@ -82,6 +82,13 @@ var ( ErrResizeNotRunning = errors.New("no resize job currently running") ) +// BadRequestError wraps an error value to signify that a request could not be +// read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest +// would be returned. +type BadRequestError struct { + error +} + // Regular expression to validate index and frame names. var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) diff --git a/test/cluster.go b/test/cluster.go index 8362ca964..308a7e69c 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -129,7 +129,7 @@ func (t *TestCluster) SetBit(index, frame, view string, rowID, colID uint64, x * // Determine which node should receive the SetBit. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. slice := colID / pilosa.SliceWidth - nodes := c0.FragmentNodes(index, slice) + nodes := c0.SliceNodes(index, slice) for _, node := range nodes { c := t.clusterByID(node.ID) @@ -153,7 +153,7 @@ func (t *TestCluster) SetFieldValue(index, frame string, columnID uint64, name s // Determine which node should receive the SetFieldValue. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. slice := columnID / pilosa.SliceWidth - nodes := c0.FragmentNodes(index, slice) + nodes := c0.SliceNodes(index, slice) for _, node := range nodes { c := t.clusterByID(node.ID) From 47a5ed84bd70e78d480b6d308ba2589c044b3c81 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 12 Apr 2018 14:46:44 -0500 Subject: [PATCH 24/79] add executor min/max tests. fix related bugs. --- client_test.go | 2 +- executor.go | 4 +-- executor_test.go | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/client_test.go b/client_test.go index 4db9156c5..2fde7a7c7 100644 --- a/client_test.go +++ b/client_test.go @@ -355,7 +355,7 @@ func TestClient_ImportValue(t *testing.T) { if err != nil { t.Fatal(err) } - min, cnt, err = frame.FieldMin(filter, fld.Name) // TODO: change this to use the client + min, cnt, err = frame.FieldMin(filter, fld.Name) if err != nil { t.Fatal(err) } diff --git a/executor.go b/executor.go index 022e48dda..83f35e7d3 100644 --- a/executor.go +++ b/executor.go @@ -1782,7 +1782,7 @@ type MinCount struct { // Smaller returns the smaller of the two MinCounts. func (mc *MinCount) Smaller(other MinCount) MinCount { - if mc.Count == 0 || other.Count < mc.Count { + if mc.Count == 0 || (other.Min < mc.Min && other.Count > 0) { return other } return MinCount{ @@ -1799,7 +1799,7 @@ type MaxCount struct { // Larger returns the larger of the two MaxCounts. func (mc *MaxCount) Larger(other MaxCount) MaxCount { - if mc.Count == 0 || other.Count < mc.Count { + if mc.Count == 0 || (other.Max > mc.Max && other.Count > 0) { return other } return MaxCount{ diff --git a/executor_test.go b/executor_test.go index f2e0d210d..d5fa6e5d6 100644 --- a/executor_test.go +++ b/executor_test.go @@ -599,6 +599,98 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { } } +// Ensure Min() and Max() queries can be executed. +func TestExecutor_Execute_MinMax(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ + RangeEnabled: true, + Fields: []*pilosa.Field{ + {Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100}, + }, + }); err != nil { + t.Fatal(err) + } + + if _, err := e.Execute(context.Background(), "i", test.MustParse(` + SetBit(frame=f, row=0, col=0) + SetBit(frame=f, row=0, col=3) + SetBit(frame=f, row=0, col=`+strconv.Itoa(SliceWidth+1)+`) + SetBit(frame=f, row=1, col=1) + SetBit(frame=f, row=2, col=`+strconv.Itoa(SliceWidth+2)+`) + + SetFieldValue(frame=f, foo=20, col=0) + SetFieldValue(frame=f, foo=-5, col=1) + SetFieldValue(frame=f, foo=-5, col=2) + SetFieldValue(frame=f, foo=10, col=3) + SetFieldValue(frame=f, foo=30, col=`+strconv.Itoa(SliceWidth)+`) + SetFieldValue(frame=f, foo=40, col=`+strconv.Itoa(SliceWidth+2)+`) + SetFieldValue(frame=f, foo=50, col=`+strconv.Itoa((5*SliceWidth)+100)+`) + SetFieldValue(frame=f, foo=60, col=`+strconv.Itoa(SliceWidth+1)+`) + `), nil, nil); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: -5, cnt: 2}, + {filter: `Bitmap(frame=f, row=0)`, exp: 10, cnt: 1}, + {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(frame=f, field=foo)` + } else { + pql = fmt.Sprintf(`Min(%s, frame=f, field=foo)`, tt.filter) + } + if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], pilosa.MinCount{Min: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 60, cnt: 1}, + {filter: `Bitmap(frame=f, row=0)`, exp: 60, cnt: 1}, + {filter: `Bitmap(frame=f, row=1)`, exp: -5, cnt: 1}, + {filter: `Bitmap(frame=f, row=2)`, exp: 40, cnt: 1}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(frame=f, field=foo)` + } else { + pql = fmt.Sprintf(`Max(%s, frame=f, field=foo)`, tt.filter) + } + if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result[0], pilosa.MaxCount{Max: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) +} + // Ensure a Sum() query can be executed. func TestExecutor_Execute_Sum(t *testing.T) { hldr := test.MustOpenHolder() From 8ec801a7322ce64fce3bc34d16c646b67e837cc1 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 12 Apr 2018 16:37:49 -0500 Subject: [PATCH 25/79] use io.WriterTo instead of custom --- api.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/api.go b/api.go index f57d159f2..c447ef9f2 100644 --- a/api.go +++ b/api.go @@ -271,15 +271,10 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) return api.Cluster.SliceNodes(indexName, slice) } -// WriterTo is an interface for any Object which knows how to serialize itself to an io.Writer -type WriterTo interface { - WriteTo(w io.Writer) (n int64, err error) -} - // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. -func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (WriterTo, error) { +func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { From a04ae99fb30d44e9ebc3f3a59b849f125f751084 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 13 Apr 2018 03:41:59 +0300 Subject: [PATCH 26/79] updated cluster tutorial with feedback --- docs/tutorials.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index 4d1a06c42..5e0d5376d 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -70,7 +70,7 @@ We now should have `pilosa.local.gossip32` in the current directory with 32 rand Pilosa supports passing configuration items using the command line, environment variables or a configuration file. We will use the last option in this tutorial and create three configuration files for our three nodes. -One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. +One of the nodes in the cluster must be chosen as the *coordinator*. We choose the first node as the coordinator in this tutorial. The coordinator is only important during cluster resizing operations, and otherwise acts like any other node in the cluster. In the future, the coordinator will be chosen transparently by distributed consensus, and this option will be deprecated. Create `node1.config.toml` in the project directory and paste the following in it: From 3f303d1098e6173d9f0af0bc031772f99525d67d Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 12 Apr 2018 19:52:48 -0500 Subject: [PATCH 27/79] remove global defaults from config.go these were occaisionally referenced elsewhere in the codebase - in all but one case, there were workarounds that are actually better I think. In the one case there wasn't I created a single top level DefaultConfig object which is instantiated with all the default values and can be referred to if necessary. There was a bug in fragment.go with the way MaxWritesPerRequest was treated if it was 0. Elsewhere, 0 meant no limit, but here, it would have caused a division by 0. Changed the default metrics provider from "nop" to "none", although "nop" will still work. Previously, any value other than "statsd" or "expvar" was treated as "nop", but I've changed this behavior to return an error if an invalid string is provided. I think this is better behavior, because in the case that someone bothered to change the default, they were probably interested in actually getting stats, and might be annoyed when it silently failed. --- cluster.go | 9 +- config.go | 215 ++++++++++++++++-------------------- config_test.go | 2 +- ctl/generate_config_test.go | 3 +- ctl/import_test.go | 2 +- docs/configuration.md | 2 +- fragment.go | 10 +- server.go | 5 +- server/server.go | 14 +-- 9 files changed, 118 insertions(+), 144 deletions(-) diff --git a/cluster.go b/cluster.go index abf753892..c7945d938 100644 --- a/cluster.go +++ b/cluster.go @@ -275,11 +275,10 @@ type Cluster struct { // NewCluster returns a new instance of Cluster with defaults. func NewCluster() *Cluster { return &Cluster{ - Hasher: &jmphasher{}, - PartitionN: DefaultPartitionN, - ReplicaN: DefaultReplicaN, - MaxWritesPerRequest: DefaultMaxWritesPerRequest, - EventReceiver: NopEventReceiver, + Hasher: &jmphasher{}, + PartitionN: DefaultPartitionN, + ReplicaN: DefaultReplicaN, + EventReceiver: NopEventReceiver, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel jobs: make(map[int64]*ResizeJob), diff --git a/config.go b/config.go index 84f199dc3..c03f03ac3 100644 --- a/config.go +++ b/config.go @@ -25,93 +25,8 @@ const ( ClusterGossip = "gossip" ) -const ( - // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" - - // DefaultHost is the default hostname to use. - DefaultHost = "localhost" - - // DefaultPort is the default port to use with the hostname. - DefaultPort = "10101" - - // DefaultClusterDisabled sets the node intercommunication method. - DefaultClusterDisabled = false - - // DefaultMetrics sets the internal metrics to no-op. - DefaultMetrics = "nop" - - // DefaultMaxWritesPerRequest is the default number of writes per request. - DefaultMaxWritesPerRequest = 5000 - - // Gossip config based on memberlist.Config. - - // Port indicates the port to which pilosa should bind for internal state sharing. - DefaultGossipPort = "14000" - - // StreamTimeout is the timeout for establishing a stream connection with - // a remote node for a full state sync, and for stream read and write - // operations. Maps to memberlist TCPTimeout. - DefaultGossipStreamTimeout = 10 * time.Second - - // SuspicionMult is the multiplier for determining the time an - // inaccessible node is considered suspect before declaring it dead. - // The actual timeout is calculated using the formula: - // - // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval - // - // This allows the timeout to scale properly with expected propagation - // delay with a larger cluster size. The higher the multiplier, the longer - // an inaccessible node is considered part of the cluster before declaring - // it dead, giving that suspect node more time to refute if it is indeed - // still alive. - DefaultGossipSuspicionMult = 4 - - // PushPullInterval is the interval between complete state syncs. - // Complete state syncs are done with a single node over TCP and are - // quite expensive relative to standard gossiped messages. Setting this - // to zero will disable state push/pull syncs completely. - // - // Setting this interval lower (more frequent) will increase convergence - // speeds across larger clusters at the expense of increased bandwidth - // usage. - DefaultGossipPushPullInterval = 30 * time.Second - - // ProbeInterval and ProbeTimeout are used to configure probing behavior - // for memberlist. - // - // ProbeInterval is the interval between random node probes. Setting - // this lower (more frequent) will cause the memberlist cluster to detect - // failed nodes more quickly at the expense of increased bandwidth usage. - // - // ProbeTimeout is the timeout to wait for an ack from a probed node - // before assuming it is unhealthy. This should be set to 99-percentile - // of RTT (round-trip time) on your network. - DefaultGossipProbeInterval = 1 * time.Second - DefaultGossipProbeTimeout = 500 * time.Millisecond - - // Interval and Nodes are used to configure the gossip - // behavior of memberlist. - // - // Interval is the interval between sending messages that need - // to be gossiped that haven't been able to piggyback on probing messages. - // If this is set to zero, non-piggyback gossip is disabled. By lowering - // this value (more frequent) gossip messages are propagated across - // the cluster more quickly at the expense of increased bandwidth. - // - // Nodes is the number of random nodes to send gossip messages to - // per Interval. Increasing this number causes the gossip messages - // to propagate across the cluster more quickly at the expense of - // increased bandwidth. - // - // ToTheDeadTime is the interval after which a node has died that - // we will still try to gossip to it. This gives it a chance to refute. - DefaultGossipInterval = 200 * time.Millisecond - DefaultGossipNodes = 3 - DefaultGossipToTheDeadTime = 30 * time.Second - - DefaultMetricPollInterval = 0 * time.Minute -) +// DefaultConfig is a Config structure that holds all the default values. +var DefaultConfig = NewConfig() // TLSConfig contains TLS configuration type TLSConfig struct { @@ -125,20 +40,27 @@ type TLSConfig struct { // Config represents the configuration for the command. type Config struct { + // DataDir is the default data directory. DataDir string `toml:"data-dir"` - Bind string `toml:"bind"` + // Bind is the host:port on which Pilosa will listen. + Bind string `toml:"bind"` - // Limits the number of mutating commands that can be in a single request to - // the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs. + // MaxWritesPerRequest limits the number of mutating commands that can be in + // a single request to the server. This includes SetBit, ClearBit, + // SetRowAttrs & SetColumnAttrs. MaxWritesPerRequest int `toml:"max-writes-per-request"` + // LogPath configures where Pilosa will write logs. LogPath string `toml:"log-path"` - Verbose bool `toml:"verbose"` + + // Verbose toggles verbose logging which can be useful for debugging. + Verbose bool `toml:"verbose"` // TLS TLS TLSConfig Cluster struct { + // Disabled controls whether clustering functionality is enabled. Disabled bool `toml:"disabled"` Coordinator bool `toml:"coordinator"` ReplicaN int `toml:"replicas"` @@ -146,18 +68,69 @@ type Config struct { LongQueryTime Duration `toml:"long-query-time"` } `toml:"cluster"` + // Gossip config is based around memberlist.Config. Gossip struct { - Port string `toml:"port"` - Seeds []string `toml:"seeds"` - Key string `toml:"key"` - StreamTimeout Duration `toml:"stream-timeout"` - SuspicionMult int `toml:"suspicion-mult"` + // Port indicates the port to which pilosa should bind for internal state sharing. + Port string `toml:"port"` + Seeds []string `toml:"seeds"` + Key string `toml:"key"` + // StreamTimeout is the timeout for establishing a stream connection with + // a remote node for a full state sync, and for stream read and write + // operations. Maps to memberlist TCPTimeout. + StreamTimeout Duration `toml:"stream-timeout"` + // SuspicionMult is the multiplier for determining the time an + // inaccessible node is considered suspect before declaring it dead. + // The actual timeout is calculated using the formula: + // + // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval + // + // This allows the timeout to scale properly with expected propagation + // delay with a larger cluster size. The higher the multiplier, the longer + // an inaccessible node is considered part of the cluster before declaring + // it dead, giving that suspect node more time to refute if it is indeed + // still alive. + SuspicionMult int `toml:"suspicion-mult"` + // PushPullInterval is the interval between complete state syncs. + // Complete state syncs are done with a single node over TCP and are + // quite expensive relative to standard gossiped messages. Setting this + // to zero will disable state push/pull syncs completely. + // + // Setting this interval lower (more frequent) will increase convergence + // speeds across larger clusters at the expense of increased bandwidth + // usage. PushPullInterval Duration `toml:"push-pull-interval"` - ProbeTimeout Duration `toml:"probe-timeout"` - ProbeInterval Duration `toml:"probe-interval"` - Nodes int `toml:"nodes"` - Interval Duration `toml:"interval"` - ToTheDeadTime Duration `toml:"to-the-dead-time"` + // ProbeInterval and ProbeTimeout are used to configure probing behavior + // for memberlist. + // + // ProbeInterval is the interval between random node probes. Setting + // this lower (more frequent) will cause the memberlist cluster to detect + // failed nodes more quickly at the expense of increased bandwidth usage. + // + // ProbeTimeout is the timeout to wait for an ack from a probed node + // before assuming it is unhealthy. This should be set to 99-percentile + // of RTT (round-trip time) on your network. + ProbeInterval Duration `toml:"probe-interval"` + ProbeTimeout Duration `toml:"probe-timeout"` + + // Interval and Nodes are used to configure the gossip + // behavior of memberlist. + // + // Interval is the interval between sending messages that need + // to be gossiped that haven't been able to piggyback on probing messages. + // If this is set to zero, non-piggyback gossip is disabled. By lowering + // this value (more frequent) gossip messages are propagated across + // the cluster more quickly at the expense of increased bandwidth. + // + // Nodes is the number of random nodes to send gossip messages to + // per Interval. Increasing this number causes the gossip messages + // to propagate across the cluster more quickly at the expense of + // increased bandwidth. + // + // ToTheDeadTime is the interval after which a node has died that + // we will still try to gossip to it. This gives it a chance to refute. + Interval Duration `toml:"interval"` + Nodes int `toml:"nodes"` + ToTheDeadTime Duration `toml:"to-the-dead-time"` } `toml:"gossip"` AntiEntropy struct { @@ -165,51 +138,55 @@ type Config struct { } `toml:"anti-entropy"` Metric struct { - Service string `toml:"service"` + // Service can be statsd, expvar, or none. + Service string `toml:"service"` + // Host tells the statsd client where to write. Host string `toml:"host"` PollInterval Duration `toml:"poll-interval"` - Diagnostics bool `toml:"diagnostics"` + // Diagnostics toggles sending some limited diagnostic information to + // Pilosa's developers. + Diagnostics bool `toml:"diagnostics"` } `toml:"metric"` } // NewConfig returns an instance of Config with default options. func NewConfig() *Config { c := &Config{ - DataDir: DefaultDataDir, - Bind: ":" + DefaultPort, - MaxWritesPerRequest: DefaultMaxWritesPerRequest, + DataDir: "~/.pilosa", + Bind: ":10101", + MaxWritesPerRequest: 5000, // LogPath: "", // Verbose: false, TLS: TLSConfig{}, } // Cluster config. - c.Cluster.Disabled = DefaultClusterDisabled + c.Cluster.Disabled = false // c.Cluster.Coordinator = false c.Cluster.ReplicaN = DefaultReplicaN c.Cluster.Hosts = []string{} c.Cluster.LongQueryTime = Duration(time.Minute) // Gossip config. - // c.Gossip.Port = "" + c.Gossip.Port = "14000" // c.Gossip.Seeds = []string{} // c.Gossip.Key = "" - c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout) - c.Gossip.SuspicionMult = DefaultGossipSuspicionMult - c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval) - c.Gossip.ProbeTimeout = Duration(DefaultGossipProbeTimeout) - c.Gossip.ProbeInterval = Duration(DefaultGossipProbeInterval) - c.Gossip.Nodes = DefaultGossipNodes - c.Gossip.Interval = Duration(DefaultGossipInterval) - c.Gossip.ToTheDeadTime = Duration(DefaultGossipToTheDeadTime) + c.Gossip.StreamTimeout = Duration(10 * time.Second) + c.Gossip.SuspicionMult = 4 + c.Gossip.PushPullInterval = Duration(30 * time.Second) + c.Gossip.ProbeInterval = Duration(1 * time.Second) + c.Gossip.ProbeTimeout = Duration(500 * time.Millisecond) + c.Gossip.Interval = Duration(200 * time.Millisecond) + c.Gossip.Nodes = 3 + c.Gossip.ToTheDeadTime = Duration(30 * time.Second) // AntiEntropy config. - c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) + c.AntiEntropy.Interval = Duration(10 * time.Minute) // Metric config. - c.Metric.Service = DefaultMetrics + c.Metric.Service = "none" // c.Metric.Host = "" - c.Metric.PollInterval = Duration(DefaultMetricPollInterval) + c.Metric.PollInterval = Duration(0 * time.Minute) c.Metric.Diagnostics = true return c diff --git a/config_test.go b/config_test.go index 1359e2e31..c07486ddb 100644 --- a/config_test.go +++ b/config_test.go @@ -25,7 +25,7 @@ import ( func Test_NewConfig(t *testing.T) { c := pilosa.NewConfig() - if c.Cluster.Disabled != pilosa.DefaultClusterDisabled { + if c.Cluster.Disabled { t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled) } diff --git a/ctl/generate_config_test.go b/ctl/generate_config_test.go index 801ab4055..56f392bba 100644 --- a/ctl/generate_config_test.go +++ b/ctl/generate_config_test.go @@ -17,7 +17,6 @@ package ctl import ( "bytes" "context" - "github.com/pilosa/pilosa" "io" "os" "strings" @@ -35,7 +34,7 @@ func TestGenerateConfigCommand_Run(t *testing.T) { io.Copy(&buf, r) if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), pilosa.DefaultHost) { + } else if !strings.Contains(buf.String(), "localhost:10101") { t.Fatalf("Unexpected config: %s", buf.String()) } } diff --git a/ctl/import_test.go b/ctl/import_test.go index bd259e342..55ae6d243 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -134,7 +134,7 @@ func TestImportCommand_InvalidFile(t *testing.T) { buf := bytes.Buffer{} stdin, stdout, stderr := GetIO(buf) cm := NewImportCommand(stdin, stdout, stderr) - cm.Host = pilosa.DefaultHost + cm.Host = "anyhost" cm.Index = "i" cm.Frame = "f" file, err := ioutil.TempFile("", "import.csv") diff --git a/docs/configuration.md b/docs/configuration.md index 4d71dee8e..2528299a4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -213,7 +213,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h ``` #### Metric Service -* Description: Which stats service to use. Choose from [statsd, expvar]. +* Description: Which stats service to use. Choose from [statsd, expvar, none]. * Flag: `--metric.service=statsd` * Env: `PILOSA_METRIC_SERVICE=statsd' * Config: diff --git a/fragment.go b/fragment.go index c44a07380..56342690c 100644 --- a/fragment.go +++ b/fragment.go @@ -1832,15 +1832,19 @@ func (s *FragmentSyncer) syncBlock(id int) error { // Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest. total := len(set.ColumnIDs) + len(clear.ColumnIDs) - buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(s.Cluster.MaxWritesPerRequest)))) + maxWrites := s.Cluster.MaxWritesPerRequest + if maxWrites <= 0 { + maxWrites = 5000 + } + buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites)))) // Only sync the standard block. for j := 0; j < len(set.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "SetBit(frame=%q, row=%d, col=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) count++ } for j := 0; j < len(clear.ColumnIDs); j++ { - fmt.Fprintf(&(buffers[count/s.Cluster.MaxWritesPerRequest]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) + fmt.Fprintf(&(buffers[count/maxWrites]), "ClearBit(frame=%q, row=%d, col=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) count++ } diff --git a/server.go b/server.go index 21f974ad8..57ae423df 100644 --- a/server.go +++ b/server.go @@ -37,8 +37,7 @@ import ( // Default server settings. const ( - DefaultAntiEntropyInterval = 10 * time.Minute - DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" + DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) // Ensure Server implements interfaces. @@ -108,7 +107,7 @@ func NewServer() *Server { NewAttrStore: NewNopAttrStore, - AntiEntropyInterval: DefaultAntiEntropyInterval, + AntiEntropyInterval: time.Duration(DefaultConfig.AntiEntropy.Interval), MetricInterval: 0, DiagnosticInterval: 0, diff --git a/server/server.go b/server/server.go index 62c7a70a5..7f66d31ed 100644 --- a/server/server.go +++ b/server/server.go @@ -20,7 +20,6 @@ package server import ( - "errors" "fmt" "io" "log" @@ -40,6 +39,7 @@ import ( "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/statik" "github.com/pilosa/pilosa/statsd" + "github.com/pkg/errors" ) func init() { @@ -255,13 +255,7 @@ func (m *Command) SetupNetworking() error { return nil } - // Set internal port (string). - gossipPortStr := pilosa.DefaultGossipPort - if m.Config.Gossip.Port != "" { - gossipPortStr = m.Config.Gossip.Port - } - - gossipPort, err := strconv.Atoi(gossipPortStr) + gossipPort, err := strconv.Atoi(m.Config.Gossip.Port) if err != nil { return err } @@ -318,7 +312,9 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { return pilosa.NewExpvarStatsClient(), nil case "statsd": return statsd.NewStatsClient(host) - default: + case "nop", "none": return pilosa.NopStatsClient, nil + default: + return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].") } } From 0c35094bfc43b311239fc1da47a550445ebde44b Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 13 Apr 2018 10:21:47 -0500 Subject: [PATCH 28/79] few more docs tweaks --- docs/configuration.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2528299a4..2d9d109b9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -197,12 +197,12 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h ```toml [profile] - cpu = "/path/to/somewhere" + cpu = "/path/to/somewhere" ``` #### Profile CPU Time -* Description: Amount of time to collect cpu profiling data if `profile.cpu` is set. +* Description: Amount of time to collect cpu profiling data at startup if `profile.cpu` is set. * Flag: `--profile.cpu-time="30s"` * Env: `PILOSA_PROFILE_CPU_TIME="30s" * Config: @@ -236,7 +236,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Metric Poll Interval -* Description: Polling interval for runtime metrics. +* Description: Rate at which runtime metrics (such as open file handles and memory usage) are collected. * Flag: `metric.poll-interval=”0m15s”` * Env: `PILOSA_METRIC_POLL_INTERVAL=0m15s` * Config: @@ -248,7 +248,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Metric Diagnostics -* Description: Enable diagnostic reporting. To disable diagnostics set to false. +* Description: Enable reporting of limited usage statistics to Pilosa developers. To disable, set to false. * Flag: `metric.diagnostics` * Env: `PILOSA_METRIC_DIAGNOSTICS` * Config: From 7233597f35951eb5827cc3d2a059b190213dba4c Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Sun, 15 Apr 2018 17:20:14 -0500 Subject: [PATCH 29/79] remove DefaultConfig global (only used once). Fix data-dir comment --- config.go | 6 ++---- server.go | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/config.go b/config.go index c03f03ac3..483965916 100644 --- a/config.go +++ b/config.go @@ -25,9 +25,6 @@ const ( ClusterGossip = "gossip" ) -// DefaultConfig is a Config structure that holds all the default values. -var DefaultConfig = NewConfig() - // TLSConfig contains TLS configuration type TLSConfig struct { // CertificatePath contains the path to the certificate (.crt or .pem file) @@ -40,7 +37,8 @@ type TLSConfig struct { // Config represents the configuration for the command. type Config struct { - // DataDir is the default data directory. + // DataDir is the directory where Pilosa stores both indexed data and + // running state such as cluster topology information. DataDir string `toml:"data-dir"` // Bind is the host:port on which Pilosa will listen. Bind string `toml:"bind"` diff --git a/server.go b/server.go index 57ae423df..bb629b66a 100644 --- a/server.go +++ b/server.go @@ -107,7 +107,7 @@ func NewServer() *Server { NewAttrStore: NewNopAttrStore, - AntiEntropyInterval: time.Duration(DefaultConfig.AntiEntropy.Interval), + AntiEntropyInterval: time.Duration(NewConfig().AntiEntropy.Interval), MetricInterval: 0, DiagnosticInterval: 0, From fe0ba6280f27f998e34502c8f81f426714e3e6ca Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 16 Apr 2018 16:08:04 -0500 Subject: [PATCH 30/79] deprecate RangeEnabled, but leave in API the RangeEnabled option now has no effect, but it still exists in the API. A few tests still use it to ensure this. This would only be considered a breaking change if someone was relying on Pilosa to enforce the RangeEnabled: false option to prevent fields being created in certain frames. This seems unlikely. --- client_test.go | 2 +- cluster_test.go | 1 - cmd/import.go | 2 +- ctl/import_test.go | 6 ++---- diagnostics.go | 6 ++---- docs/api-reference.md | 4 ++-- docs/tutorials.md | 1 - executor_test.go | 7 ------- frame.go | 35 +---------------------------------- frame_test.go | 5 ----- handler.go | 2 -- handler_test.go | 23 +++++++++++------------ index.go | 11 +---------- index_test.go | 24 ++++++++++-------------- pilosa.go | 23 ++++++++++------------- 15 files changed, 41 insertions(+), 111 deletions(-) diff --git a/client_test.go b/client_test.go index cdbaba788..4bd0bade6 100644 --- a/client_test.go +++ b/client_test.go @@ -310,7 +310,7 @@ func TestClient_ImportValue(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true, Fields: []*pilosa.Field{&fld}}) + frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{Fields: []*pilosa.Field{&fld}}) if err != nil { t.Fatal(err) } diff --git a/cluster_test.go b/cluster_test.go index c5c01d58d..6c5d41829 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -427,7 +427,6 @@ func TestCluster_ResizeStates(t *testing.T) { // Add Field Data to node0. if err := tc.CreateFrame("i", "fields", pilosa.FrameOptions{ InverseEnabled: false, - RangeEnabled: true, //CacheType: pilosa.CacheTypeNone, Fields: []*pilosa.Field{ { diff --git a/cmd/import.go b/cmd/import.go index 8692fa318..b6b78bdbc 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -63,7 +63,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index (DEPRECATED. This feature will be removed in a future version. Set time quantum of each frame instead.)") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame") - flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "Enabled range encoded frame") + flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.") flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked") flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame") ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify) diff --git a/ctl/import_test.go b/ctl/import_test.go index 55ae6d243..6cbc8271a 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -86,9 +86,7 @@ func TestImportCommand_Run(t *testing.T) { } // Ensure that the ImportValue path runs (note: we have specified a value -// for cm.Field. Because the handler doesn't return errors (it sends them -// to the logger), we don't get an error returned at `cm.Run()` even though -// we haven't setup frame `f` to be RangeEnabled. +// for cm.Field.) func TestImportCommand_RunValue(t *testing.T) { buf := bytes.Buffer{} @@ -117,7 +115,7 @@ func TestImportCommand_RunValue(t *testing.T) { cm.Host = s.Host() http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"rangeEnabled": true, "fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) cm.Index = "i" cm.Frame = "f" diff --git a/diagnostics.go b/diagnostics.go index 5d3f6699b..57423d0a5 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -219,10 +219,8 @@ func (d *DiagnosticsCollector) EnrichWithSchemaProperties() { numIndexes += 1 for _, frame := range index.Frames() { numFrames += 1 - if frame.rangeEnabled { - if fields, err := frame.GetFields(); err == nil { - bsiFieldCount += len(fields) - } + if fields, err := frame.GetFields(); err == nil { + bsiFieldCount += len(fields) } if frame.TimeQuantum() != "" { timeQuantumEnabled = true diff --git a/docs/api-reference.md b/docs/api-reference.md index 446235cdc..0c0314f85 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -106,7 +106,7 @@ The request payload is in JSON, and may contain the `options` field. The `option * `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`. * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. -* `rangeEnabled` (boolean): Enables range-encoded fields in this frame. +* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields. * `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding). Each individual `field` contains the following: @@ -130,7 +130,7 @@ curl localhost:10101/index/user/frame/language \ ``` request curl localhost:10101/index/repository/frame/stats \ -X POST \ - -d '{"rangeEnabled": true, "fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' + -d '{"fields": [{"name": "pullrequests", "type": "int", "min": 0, "max": 1000000}]}' ``` ``` response {} diff --git a/docs/tutorials.md b/docs/tutorials.md index 5e0d5376d..601825fa9 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -259,7 +259,6 @@ In addition to storing rows of bits, a frame can also contain fields that store curl localhost:10101/index/patients/frame/measurements \ -X POST \ -d '{"options":{ - "rangeEnabled": true, "fields": [ {"name": "age", "type": "int", "min": 0, "max": 120}, {"name": "weight", "type": "int", "min": 0, "max": 500}, diff --git a/executor_test.go b/executor_test.go index f2e0d210d..1f8a38b99 100644 --- a/executor_test.go +++ b/executor_test.go @@ -280,7 +280,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { // Create frames. index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 50}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 1, Max: 2}, @@ -330,7 +329,6 @@ func TestExecutor_Execute_SetFieldValue(t *testing.T) { defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}, }, @@ -611,7 +609,6 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, @@ -621,7 +618,6 @@ func TestExecutor_Execute_Sum(t *testing.T) { } if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, }, @@ -723,7 +719,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100}, {Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000}, @@ -733,7 +728,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := idx.CreateFrame("other", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000}, }, @@ -742,7 +736,6 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { } if _, err := idx.CreateFrame("edge", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: -100, Max: 100}, }, diff --git a/frame.go b/frame.go index def41d1b1..6ed76d61c 100644 --- a/frame.go +++ b/frame.go @@ -33,7 +33,6 @@ import ( const ( DefaultCacheType = CacheTypeRanked DefaultInverseEnabled = false - DefaultRangeEnabled = false // Default ranked frame cache DefaultCacheSize = 50000 @@ -59,7 +58,6 @@ type Frame struct { cacheType string cacheSize uint32 timeQuantum TimeQuantum - rangeEnabled bool fields []*Field Logger Logger @@ -88,7 +86,6 @@ func NewFrame(path, index, name string) (*Frame, error) { cacheType: DefaultCacheType, cacheSize: DefaultCacheSize, //timeQuantum - rangeEnabled: DefaultRangeEnabled, //fields Logger: NopLogger, @@ -145,11 +142,6 @@ func (f *Frame) InverseEnabled() bool { return f.inverseEnabled } -// RangeEnabled returns true if range fields can be stored on this frame. -func (f *Frame) RangeEnabled() bool { - return f.rangeEnabled -} - // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 func (f *Frame) SetCacheSize(v uint32) error { @@ -188,7 +180,6 @@ func (f *Frame) Options() FrameOptions { func (f *Frame) options() FrameOptions { return FrameOptions{ InverseEnabled: f.inverseEnabled, - RangeEnabled: f.rangeEnabled, CacheType: f.cacheType, CacheSize: f.cacheSize, TimeQuantum: f.timeQuantum, @@ -268,7 +259,6 @@ func (f *Frame) loadMeta() error { f.cacheType = DefaultCacheType f.cacheSize = DefaultCacheSize f.timeQuantum = "" - f.rangeEnabled = DefaultRangeEnabled //f.fields return nil } else if err != nil { @@ -287,7 +277,6 @@ func (f *Frame) loadMeta() error { } f.cacheSize = pb.CacheSize f.timeQuantum = TimeQuantum(pb.TimeQuantum) - f.rangeEnabled = pb.RangeEnabled f.fields = decodeFields(pb.Fields) return nil @@ -365,11 +354,6 @@ func (f *Frame) CreateField(field *Field) error { f.mu.Lock() defer f.mu.Unlock() - // Ensure frame supports fields. - if !f.RangeEnabled() { - return ErrFrameFieldsNotAllowed - } - // Append field. if err := f.addField(field); err != nil { return err @@ -402,11 +386,6 @@ func (f *Frame) GetFields() ([]*Field, error) { f.mu.RLock() defer f.mu.RUnlock() - // Ensure the frame supports fields. - if !f.RangeEnabled() { - return nil, ErrFrameFieldsNotAllowed - } - err := f.loadMeta() if err != nil { return nil, err @@ -420,11 +399,6 @@ func (f *Frame) DeleteField(name string) error { f.mu.Lock() defer f.mu.Unlock() - // Ensure frame supports fields. - if !f.RangeEnabled() { - return ErrFrameFieldsNotAllowed - } - // Remove field. if err := f.deleteField(name); err != nil { return err @@ -890,11 +864,6 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // ImportValue bulk imports range-encoded value data. func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { - // Verify that this frame is range-encoded. - if !f.RangeEnabled() { - return fmt.Errorf("Frame not RangeEnabled: %s", f.name) - } - viewName := ViewFieldPrefix + fieldName // Get the field so we know bitDepth. field := f.Field(fieldName) @@ -991,7 +960,7 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { InverseEnabled bool `json:"inverseEnabled,omitempty"` - RangeEnabled bool `json:"rangeEnabled,omitempty"` + RangeEnabled bool `json:"rangeEnabled,omitempty"` // deprecated, will be removed CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` @@ -1009,7 +978,6 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { } return &internal.FrameMeta{ InverseEnabled: o.InverseEnabled, - RangeEnabled: o.RangeEnabled, CacheType: o.CacheType, CacheSize: o.CacheSize, TimeQuantum: string(o.TimeQuantum), @@ -1023,7 +991,6 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { } return &FrameOptions{ InverseEnabled: options.InverseEnabled, - RangeEnabled: options.RangeEnabled, CacheType: options.CacheType, CacheSize: options.CacheSize, TimeQuantum: TimeQuantum(options.TimeQuantum), diff --git a/frame_test.go b/frame_test.go index 2107bd703..78014bbce 100644 --- a/frame_test.go +++ b/frame_test.go @@ -77,7 +77,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 20, Max: 25}, @@ -123,7 +122,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, }, @@ -161,7 +159,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 0, Max: 30}, }, @@ -181,7 +178,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, }, @@ -201,7 +197,6 @@ func TestFrame_SetFieldValue(t *testing.T) { defer idx.Close() f, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 20, Max: 30}, }, diff --git a/handler.go b/handler.go index bc6c625ef..5e105073a 100644 --- a/handler.go +++ b/handler.go @@ -788,8 +788,6 @@ func (h *Handler) handleGetFrameFields(w http.ResponseWriter, r *http.Request) { fallthrough case ErrFrameNotFound: http.Error(w, err.Error(), http.StatusNotFound) - case ErrFrameFieldsNotAllowed: - http.Error(w, err.Error(), http.StatusBadRequest) default: http.Error(w, err.Error(), http.StatusInternalServerError) } diff --git a/handler_test.go b/handler_test.go index 063421667..efa931970 100644 --- a/handler_test.go +++ b/handler_test.go @@ -902,7 +902,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("OK", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } @@ -927,7 +927,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("ErrInvalidFieldType", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil { + if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -949,7 +949,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("ErrInvalidFieldRange", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}); err != nil { + if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -972,8 +972,7 @@ func TestHandler_Frame_AddField(t *testing.T) { t.Run("ErrFieldAlreadyExists", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) if _, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - RangeEnabled: true, - Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}}, + Fields: []*pilosa.Field{{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}}, }); err != nil { t.Fatal(err) } @@ -1006,7 +1005,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) { t.Run("OK", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { @@ -1034,7 +1033,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) { t.Run("ErrFieldNotFound", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 0, Max: 100}); err != nil { @@ -1071,7 +1070,7 @@ func TestHandler_Frame_GetFields(t *testing.T) { t.Run("OK", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{RangeEnabled: true}) + f, err := idx.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if err := f.CreateField(&pilosa.Field{Name: "x", Type: pilosa.FieldTypeInt, Min: 1, Max: 100}); err != nil { @@ -1105,7 +1104,7 @@ func TestHandler_Frame_GetFields(t *testing.T) { t.Run("ErrFrameFieldNotAllowed", func(t *testing.T) { idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - _, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{RangeEnabled: false}) + _, err := idx.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}) resp, err := http.Get(s.URL + "/index/i/frame/f1/fields") if err != nil { @@ -1113,12 +1112,12 @@ func TestHandler_Frame_GetFields(t *testing.T) { } if err != nil { t.Fatal(err) - } else if resp.StatusCode != http.StatusBadRequest { + } else if resp.StatusCode != http.StatusOK { t.Fatalf("unexpected status code: %d", resp.StatusCode) } else if body, err := ioutil.ReadAll(resp.Body); err != nil { t.Fatal(err) - } else if strings.TrimSpace(string(body)) != `frame fields not allowed` { - t.Fatalf("unexpected body: %q", body) + } else if strings.TrimSpace(string(body)) == `frame fields not allowed` { + t.Fatalf("shouldn't get frame fields not allowed error: %q", body) } }) diff --git a/index.go b/index.go index fef4eb92c..039816322 100644 --- a/index.go +++ b/index.go @@ -404,13 +404,7 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { // Validate mutually exclusive options if ranges are enabled. if opt.RangeEnabled { - if opt.InverseEnabled { - return nil, ErrInverseRangeNotAllowed - } - } else { - if len(opt.Fields) > 0 { - return nil, ErrFrameFieldsNotAllowed - } + i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame") } // Validate fields. @@ -452,9 +446,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { } f.inverseEnabled = opt.InverseEnabled - f.rangeEnabled = opt.RangeEnabled - - f.rangeEnabled = opt.RangeEnabled // Set fields. f.fields = opt.Fields diff --git a/index_test.go b/index_test.go index c8cd238e8..439523c3c 100644 --- a/index_test.go +++ b/index_test.go @@ -99,7 +99,7 @@ func TestIndex_CreateFrame(t *testing.T) { // Create frame with schema and verify it exists. if f, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, + RangeEnabled: false, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, @@ -124,14 +124,14 @@ func TestIndex_CreateFrame(t *testing.T) { } }) - t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) { + t.Run("ErrInverseRangeAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - InverseEnabled: true, RangeEnabled: true, - }); err != pilosa.ErrInverseRangeNotAllowed { + InverseEnabled: true, + }); err != nil { t.Fatal(err) } }) @@ -141,8 +141,7 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, - CacheType: pilosa.CacheTypeRanked, + CacheType: pilosa.CacheTypeRanked, }); err != nil { t.Fatal(err) } @@ -152,15 +151,14 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, - CacheType: pilosa.CacheTypeNone, - CacheSize: uint32(5), + CacheType: pilosa.CacheTypeNone, + CacheSize: uint32(5), }); err != nil { t.Fatal(err) } }) - t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) { + t.Run("ErrFrameFieldsAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() @@ -168,7 +166,7 @@ func TestIndex_CreateFrame(t *testing.T) { Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt}, }, - }); err != pilosa.ErrFrameFieldsNotAllowed { + }); err != nil { t.Fatal(err) } }) @@ -178,7 +176,6 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "", Type: pilosa.FieldTypeInt}, }, @@ -192,7 +189,6 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "field0", Type: "bad_type"}, }, @@ -206,7 +202,7 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, + RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, }, diff --git a/pilosa.go b/pilosa.go index ffb836ae8..307f9dadc 100644 --- a/pilosa.go +++ b/pilosa.go @@ -46,19 +46,16 @@ var ( ErrInputDefinitionActionRequired = errors.New("field definitions require an action") ErrInputDefinitionNotFound = errors.New("input-definition not found") - ErrFieldNotFound = errors.New("field not found") - ErrFieldExists = errors.New("field already exists") - ErrFieldNameRequired = errors.New("field name required") - ErrInvalidFieldType = errors.New("invalid field type") - ErrInvalidFieldRange = errors.New("invalid field range") - ErrInverseRangeNotAllowed = errors.New("inverse range not allowed") - ErrRangeCacheNotAllowed = errors.New("range cache not allowed") - ErrFrameFieldsNotAllowed = errors.New("frame fields not allowed") - ErrInvalidFieldValueType = errors.New("invalid field value type") - ErrFieldValueTooLow = errors.New("field value too low") - ErrFieldValueTooHigh = errors.New("field value too high") - ErrInvalidRangeOperation = errors.New("invalid range operation") - ErrInvalidBetweenValue = errors.New("invalid value for between operation") + ErrFieldNotFound = errors.New("field not found") + ErrFieldExists = errors.New("field already exists") + ErrFieldNameRequired = errors.New("field name required") + ErrInvalidFieldType = errors.New("invalid field type") + ErrInvalidFieldRange = errors.New("invalid field range") + ErrInvalidFieldValueType = errors.New("invalid field value type") + ErrFieldValueTooLow = errors.New("field value too low") + ErrFieldValueTooHigh = errors.New("field value too high") + ErrInvalidRangeOperation = errors.New("invalid range operation") + ErrInvalidBetweenValue = errors.New("invalid value for between operation") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") From c7f2d0f675adfe3027df89b2af02f83f65bfa9fe Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 16 Apr 2018 16:16:14 -0500 Subject: [PATCH 31/79] update unreleased section of CHANGELOG with rangeEnabled deprecation --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a4d17174..950ecaad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) - Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) - Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) +- Deprecated RangeEnabled option. ([#1205](https://github.com/pilosa/pilosa/pull/1205)) ### Removed From 0a8d573fb396b05bfe06db217b08ae5552c1a6e7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 16 Apr 2018 17:12:28 -0500 Subject: [PATCH 32/79] WIP: Remove SecurityManager. Implement api restrictions in api package. --- api.go | 246 ++++++++++++++++++++++++++++++++++++++++- cluster.go | 12 +- handler.go | 59 +++------- handler_test.go | 2 +- security_manager.go | 32 ------ server.go | 4 +- server/cluster_test.go | 3 + test/cluster.go | 1 + test/handler.go | 2 - 9 files changed, 262 insertions(+), 99 deletions(-) delete mode 100644 security_manager.go diff --git a/api.go b/api.go index c447ef9f2..970766842 100644 --- a/api.go +++ b/api.go @@ -59,8 +59,30 @@ func NewAPI() *API { } } +// functionStates specifies the api functions that are valid for each +// cluster state. +var functionStates = map[string][]int{ + ClusterStateStarting: functionCommon, + ClusterStateNormal: append(functionCommon, functionNormal...), + ClusterStateResizing: append(functionCommon, functionResizing...), +} + +func (api *API) validate(f int) error { + state := api.Cluster.State() + for _, fnc := range functionStates[state] { + if f == fnc { + return nil + } + } + return fmt.Errorf("api function not allowed in state %s", state) +} + // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { + if err := api.validate(apiQuery); err != nil { + return QueryResponse{}, errors.Wrap(err, "validate api function: query") + } + resp := QueryResponse{} q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() @@ -125,6 +147,10 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { + if err := api.validate(apiCreateIndex); err != nil { + return nil, errors.Wrap(err, "validate api function: create index") + } + // Create index. index, err := api.Holder.CreateIndex(indexName, options) if err != nil { @@ -146,6 +172,10 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index // Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { + if err := api.validate(apiIndex); err != nil { + return nil, errors.Wrap(err, "validate api function: index") + } + index := api.Holder.Index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -156,6 +186,10 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { // DeleteIndex removes the named index. If the index is not found it does // nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { + if err := api.validate(apiDeleteIndex); err != nil { + return errors.Wrap(err, "validate api function: delete index") + } + // Delete index from the holder. err := api.Holder.DeleteIndex(indexName) if err != nil { @@ -176,6 +210,10 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { + if err := api.validate(apiCreateFrame); err != nil { + return nil, errors.Wrap(err, "validate api function: create frame") + } + // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -207,6 +245,10 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // found, an error is returned. If the frame is not found, it is ignored and no // action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { + if err := api.validate(apiDeleteFrame); err != nil { + return errors.Wrap(err, "validate api function: delete frame") + } + // Find index. index := api.Holder.Index(indexName) if index == nil { @@ -235,6 +277,10 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // ExportCSV encodes the fragment designated by the index,frame,view,slice as // CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { + if err := api.validate(apiExportCSV); err != nil { + return errors.Wrap(err, "validate api function: export csv") + } + // Validate that this handler owns the slice. if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { api.Logger.Printf("host does not own slice %s-%s slice:%d", api.URI, indexName, slice) @@ -267,14 +313,22 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin } // SliceNodes returns the node and all replicas which should contain a slice's data. -func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) []*Node { - return api.Cluster.SliceNodes(indexName, slice) +func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { + if err := api.validate(apiSliceNodes); err != nil { + return nil, errors.Wrap(err, "validate api function: slice nodes") + } + + return api.Cluster.SliceNodes(indexName, slice), nil } // MarshalFragment returns an object which can write the specified fragment's data // to an io.Writer. The serialized data can be read back into a fragment with // the UnmarshalFragment API call. func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { + if err := api.validate(apiMarshalFragment); err != nil { + return nil, errors.Wrap(err, "validate api function: marshal fragment") + } + // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -287,6 +341,10 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // Reader which was previously written by MarshalFragment to populate the // fragment's data. func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { + if err := api.validate(apiUnmarshalFragment); err != nil { + return errors.Wrap(err, "validate api function: unmarshal fragment") + } + // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -316,6 +374,10 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // return anything useful. Currently it returns protobuf encoded row and column // ids from a "block" which is a subdivision of a fragment. func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { + if err := api.validate(apiFragmentBlockData); err != nil { + return nil, errors.Wrap(err, "validate api function: fragment block data") + } + reqBytes, err := ioutil.ReadAll(body) if err != nil { return nil, BadRequestError{errors.Wrap(err, "read body error")} @@ -337,7 +399,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // Encode response. buf, err := proto.Marshal(&resp) if err != nil { - return nil, errors.Wrap(err, "merge block response encoding error: %s") + return nil, errors.Wrap(err, "merge block response encoding error") } return buf, nil @@ -345,6 +407,10 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { + if err := api.validate(apiFragmentBlocks); err != nil { + return nil, errors.Wrap(err, "validate api function: fragment blocks") + } + // Retrieve fragment from holder. f := api.Holder.Fragment(indexName, frameName, viewName, slice) if f == nil { @@ -359,6 +425,10 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // RestoreFrame reads all the data that this host should have for a given frame // from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { + if err := api.validate(apiRestoreFrame); err != nil { + return errors.Wrap(err, "validate api function: restore frame") + } + // Create a client for the remote cluster. client := NewInternalHTTPClientFromURI(host, api.RemoteClient) @@ -433,6 +503,10 @@ func (api *API) Hosts(ctx context.Context) []*Node { // CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { + if err := api.validate(apiCreateInputDefinition); err != nil { + return errors.Wrap(err, "validate api function: create input definition") + } + api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. Please open an issue if you need to continue using it.`) // Find index. @@ -467,6 +541,10 @@ Please open an issue if you need to continue using it.`) // InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { + if err := api.validate(apiInputDefinition); err != nil { + return nil, errors.Wrap(err, "validate api function: input definition") + } + api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) // Find index. index := api.Holder.Index(indexName) @@ -483,6 +561,10 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN // DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { + if err := api.validate(apiDeleteInputDefinition); err != nil { + return errors.Wrap(err, "validate api function: delete input definition") + } + api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) @@ -508,6 +590,10 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp // WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { + if err := api.validate(apiWriteInput); err != nil { + return errors.Wrap(err, "validate api function: write input") + } + api.Logger.Printf("WriteInput is deprecated and will be removed.") // Find index. index := api.Holder.Index(indexName) @@ -532,6 +618,10 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { + if err := api.validate(apiRecalculateCaches); err != nil { + return errors.Wrap(err, "validate api function: recalculate caches") + } + err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) if err != nil { return errors.Wrap(err, "broacasting message") @@ -542,7 +632,11 @@ func (api *API) RecalculateCaches(ctx context.Context) error { // PostClusterMessage is for internal use. It decodes a protobuf message out of // the body and forwards it to the BroadcastHandler. -func (api *API) PostClusterMessage(ctx context.Context, reqBody io.Reader) error { +func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { + if err := api.validate(apiClusterMessage); err != nil { + return errors.Wrap(err, "validate api function: cluster message") + } + // Read entire body. body, err := ioutil.ReadAll(reqBody) if err != nil { @@ -575,6 +669,10 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { + if err := api.validate(apiCreateField); err != nil { + return errors.Wrap(err, "validate api function: create field") + } + // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -601,6 +699,10 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { + if err := api.validate(apiDeleteField); err != nil { + return errors.Wrap(err, "validate api function: delete field") + } + // Retrieve frame by name. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -627,6 +729,10 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str // Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { + if err := api.validate(apiFields); err != nil { + return nil, errors.Wrap(err, "validate api function: fields") + } + index := api.Holder.index(indexName) if index == nil { return nil, ErrIndexNotFound @@ -642,6 +748,10 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { + if err := api.validate(apiViews); err != nil { + return nil, errors.Wrap(err, "validate api function: views") + } + // Retrieve views. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -655,6 +765,10 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { + if err := api.validate(apiDeleteView); err != nil { + return errors.Wrap(err, "validate api function: delete view") + } + // Retrieve frame. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -685,6 +799,10 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiIndexAttrDiff); err != nil { + return nil, errors.Wrap(err, "validate api function: index attr diff") + } + // Retrieve index from holder. index := api.Holder.Index(indexName) if index == nil { @@ -715,6 +833,10 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At } func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { + if err := api.validate(apiFrameAttrDiff); err != nil { + return nil, errors.Wrap(err, "validate api function: frame attr diff") + } + // Retrieve index from holder. f := api.Holder.Frame(indexName, frameName) if f == nil { @@ -746,6 +868,10 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { + if err := api.validate(apiImport); err != nil { + return errors.Wrap(err, "validate api function: import") + } + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { return err @@ -771,6 +897,10 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { + if err := api.validate(apiImportValue); err != nil { + return errors.Wrap(err, "validate api function: import value") + } + _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) if err != nil { return err @@ -786,6 +916,10 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { + if err := api.validate(apiModifyIndexTimeQuantum); err != nil { + return errors.Wrap(err, "validate api function: modify index time quantum") + } + // Retrieve index by name. index := api.Holder.Index(indexName) if index == nil { @@ -799,6 +933,10 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti // ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO: // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { + if err := api.validate(apiModifyFrameTimeQuantum); err != nil { + return errors.Wrap(err, "validate api function: modify frame time quantum") + } + // Retrieve index by name. frame := api.Holder.Frame(indexName, frameName) if frame == nil { @@ -933,6 +1071,10 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na // SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { + if err := api.validate(apiSetCoordinator); err != nil { + return nil, nil, errors.Wrap(err, "validate api function: set coordinator") + } + oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) newNode = api.Cluster.nodeByID(id) if newNode == nil { @@ -959,6 +1101,10 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // RemoveNode puts the cluster into the "RESIZING" state and begins the job of // removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { + if err := api.validate(apiRemoveNode); err != nil { + return nil, errors.Wrap(err, "validate api function: remove node") + } + removeNode := api.Cluster.nodeByID(id) if removeNode == nil { return nil, errors.Wrap(ErrNodeIDNotExists, "finding node to remove") @@ -974,6 +1120,10 @@ func (api *API) RemoveNode(id string) (*Node, error) { // ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { + if err := api.validate(apiResizeAbort); err != nil { + return errors.Wrap(err, "validate api function: resize abort") + } + if !api.Cluster.IsCoordinator() { return ErrNodeNotCoordinator } @@ -992,3 +1142,91 @@ func (api *API) State() string { func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } + +// API validation constants. +const ( + apiClusterMessage int = iota + apiCreateField + apiCreateFrame + apiCreateIndex + apiCreateInputDefinition + apiDeleteField + apiDeleteFrame + apiDeleteIndex + apiDeleteInputDefinition + apiDeleteView + apiExportCSV + apiFields + apiFragmentBlockData + apiFragmentBlocks + apiFrameAttrDiff + //apiHosts // not implemented + apiImport + apiImportValue + apiIndex + apiIndexAttrDiff + apiInputDefinition + //apiLocalID // not implemented + //apiLongQueryTime // not implemented + apiMarshalFragment + //apiMaxInverseSlices // not implemented + //apiMaxSlices // not implemented + apiModifyFrameTimeQuantum + apiModifyIndexTimeQuantum + apiQuery + apiRecalculateCaches + apiRemoveNode + apiResizeAbort + apiRestoreFrame + //apiSchema // not implemented + apiSetCoordinator + apiSliceNodes + //apiState // not implemented + //apiStatsWithTags // not implemented + apiUnmarshalFragment + //apiVersion // not implemented + apiViews + apiWriteInput +) + +var functionCommon = []int{ + apiClusterMessage, + apiMarshalFragment, + apiSetCoordinator, +} + +var functionResizing = []int{ + apiResizeAbort, +} + +var functionNormal = []int{ + apiCreateField, + apiCreateFrame, + apiCreateIndex, + apiCreateInputDefinition, + apiDeleteField, + apiDeleteFrame, + apiDeleteIndex, + apiDeleteInputDefinition, + apiDeleteView, + apiExportCSV, + apiFields, + apiFragmentBlockData, + apiFragmentBlocks, + apiFrameAttrDiff, + apiImport, + apiImportValue, + apiIndex, + apiIndexAttrDiff, + apiInputDefinition, + apiModifyFrameTimeQuantum, + apiModifyIndexTimeQuantum, + apiQuery, + apiRecalculateCaches, + apiRemoveNode, + apiRestoreFrame, + apiSliceNodes, + apiUnmarshalFragment, + apiViews, + apiWriteInput, +} diff --git a/cluster.go b/cluster.go index c7945d938..7e871698d 100644 --- a/cluster.go +++ b/cluster.go @@ -264,7 +264,6 @@ type Cluster struct { // Close management wg sync.WaitGroup closing chan struct{} - prefect SecurityManager Logger Logger @@ -285,8 +284,7 @@ func NewCluster() *Cluster { closing: make(chan struct{}), joining: make(chan struct{}), - Logger: NopLogger, - prefect: &NopSecurityManager{}, + Logger: NopLogger, } } @@ -433,19 +431,11 @@ func (c *Cluster) setState(state string) { var doCleanup bool switch state { - case ClusterStateResizing: - c.prefect.SetRestricted() case ClusterStateNormal: - c.prefect.SetNormal() - // Don't change routing for these states: - // - ClusterStateStarting - // If state is RESIZING -> NORMAL then run cleanup. if c.state == ClusterStateResizing { doCleanup = true } - default: - panic(fmt.Sprintf("invalid cluster state: %s", state)) } c.state = state diff --git a/handler.go b/handler.go index bc6c625ef..4e20fcb07 100644 --- a/handler.go +++ b/handler.go @@ -43,9 +43,7 @@ import ( type Handler struct { Router *mux.Router - FileSystem FileSystem - NormalRouter *mux.Router - RestrictedRouter *mux.Router + FileSystem FileSystem // The execution engine for running queries. Executor interface { @@ -83,29 +81,11 @@ func NewHandler() *Handler { FileSystem: NopFileSystem, Logger: NopLogger, } - BuildRouters(handler) + handler.Router = NewRouter(handler) handler.populateValidators() return handler } -// BuildRouters creates Gorilla Mux http routers for both normal and restricted endpoints. -func BuildRouters(handler *Handler) { - router := mux.NewRouter() - loadCommon(router, handler) - loadNormal(router, handler) - handler.NormalRouter = router - router.Use(handler.queryArgValidator) - - // Restricted router. - router = mux.NewRouter() - loadCommon(router, handler) - loadRestricted(router, handler) - handler.RestrictedRouter = router - router.Use(handler.queryArgValidator) - - handler.SetRestricted() -} - func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") @@ -138,17 +118,9 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { }) } -// SetNormal is a method of the SecurityManager interface which provides normal URI routing. -func (h *Handler) SetNormal() { - h.Router = h.NormalRouter -} - -// SetRestricted is a method of the SecurityManager interface which provides restricted URI routing. -func (h *Handler) SetRestricted() { - h.Router = h.RestrictedRouter -} - -func loadCommon(router *mux.Router, handler *Handler) { +// NewRouter creates a new mux http router. +func NewRouter(handler *Handler) *mux.Router { + router := mux.NewRouter() router.HandleFunc("/", handler.handleWebUI).Methods("GET") router.HandleFunc("/assets/{file}", handler.handleWebUI).Methods("GET") router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") @@ -162,16 +134,9 @@ func loadCommon(router *mux.Router, handler *Handler) { router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") - router.Use(handler.queryArgValidator) -} -func loadRestricted(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - router.NotFoundHandler = http.HandlerFunc(handler.reportRestricted) - router.Use(handler.queryArgValidator) -} -func loadNormal(router *mux.Router, handler *Handler) { router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.Handle("/debug/vars", expvar.Handler()).Methods("GET") @@ -212,10 +177,8 @@ func loadNormal(router *mux.Router, handler *Handler) { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") -} - -func (h *Handler) reportRestricted(w http.ResponseWriter, r *http.Request) { - http.Error(w, fmt.Sprintf("not allowed in cluster state %s", h.API.State()), http.StatusMethodNotAllowed) + router.Use(handler.queryArgValidator) + return router } func (h *Handler) methodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { @@ -1169,7 +1132,11 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } // Retrieve fragment owner nodes. - nodes := h.API.SliceNodes(r.Context(), index, slice) + nodes, err := h.API.SliceNodes(r.Context(), index, slice) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } // Write to response. if err := json.NewEncoder(w).Encode(nodes); err != nil { @@ -1727,7 +1694,7 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques return } - err := h.API.PostClusterMessage(r.Context(), r.Body) + err := h.API.ClusterMessage(r.Context(), r.Body) if err != nil { // TODO this was the previous behavior, but perhaps not everything is a bad request http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/handler_test.go b/handler_test.go index 063421667..78f9e470f 100644 --- a/handler_test.go +++ b/handler_test.go @@ -160,7 +160,7 @@ func TestHandler_ClusterResizeAbort(t *testing.T) { t.Run("No resize job", func(t *testing.T) { h := test.NewHandler() h.API.Cluster = test.NewCluster(1) - h.SetRestricted() + h.API.Cluster.SetState(pilosa.ClusterStateResizing) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil)) diff --git a/security_manager.go b/security_manager.go deleted file mode 100644 index 80b696c38..000000000 --- a/security_manager.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -// SecurityManager provides the ability to limit access to restricted endpoints -// during cluster configuration. -type SecurityManager interface { - SetRestricted() - SetNormal() -} - -// NopSecurityManager provides a no-op implementation of the SecurityManager interface. -type NopSecurityManager struct { -} - -// SetRestricted no-op. -func (sdm *NopSecurityManager) SetRestricted() {} - -// SetNormal no-op. -func (sdm *NopSecurityManager) SetNormal() {} diff --git a/server.go b/server.go index bb629b66a..049672686 100644 --- a/server.go +++ b/server.go @@ -169,10 +169,8 @@ func (s *Server) Open() error { s.Handler.API.StatusHandler = s s.Handler.API.URI = s.URI s.Handler.API.Cluster = s.Cluster - s.Handler.Executor = e - - s.Cluster.prefect = s.Handler s.Handler.API.Executor = e + s.Handler.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster diff --git a/server/cluster_test.go b/server/cluster_test.go index 04d53b438..7f4886612 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -101,6 +101,9 @@ func TestMain_SendReceiveMessage(t *testing.T) { t.Fatal(err) } + m0.Server.Cluster.SetState(pilosa.ClusterStateNormal) + m1.Server.Cluster.SetState(pilosa.ClusterStateNormal) + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Expected indexes and Frames diff --git a/test/cluster.go b/test/cluster.go index 308a7e69c..77cf496a8 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -50,6 +50,7 @@ func NewCluster(n int) *pilosa.Cluster { c.Node = c.Nodes[0] c.Coordinator = c.Nodes[0].ID + c.SetState(pilosa.ClusterStateNormal) return c } diff --git a/test/handler.go b/test/handler.go index cb90cd383..c325d3243 100644 --- a/test/handler.go +++ b/test/handler.go @@ -47,8 +47,6 @@ func NewHandler() *Handler { // Handler test messages can no-op. h.API.Broadcaster = pilosa.NopBroadcaster - h.SetNormal() - return h } From 58492c7eab45744f64deb995caef554523c9578f Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 17 Apr 2018 07:54:25 -0500 Subject: [PATCH 33/79] remove period in CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 950ecaad7..09562dc06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) - Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) - Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) -- Deprecated RangeEnabled option. ([#1205](https://github.com/pilosa/pilosa/pull/1205)) +- Deprecated RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) ### Removed From e56b8717b5d8359ee30cac3ae9b54758ab54fabe Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 17 Apr 2018 09:00:57 -0500 Subject: [PATCH 34/79] add better inverseEnabled with field test --- index_test.go | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/index_test.go b/index_test.go index 439523c3c..25e093d4b 100644 --- a/index_test.go +++ b/index_test.go @@ -128,12 +128,43 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - if _, err := index.CreateFrame("f", pilosa.FrameOptions{ + frame, err := index.CreateFrame("f", pilosa.FrameOptions{ RangeEnabled: true, InverseEnabled: true, - }); err != nil { + Fields: []*pilosa.Field{ + &pilosa.Field{ + Name: "myfield", + Type: pilosa.FieldTypeInt, + Min: -20, + Max: 100, + }, + }, + }) + if err != nil { t.Fatal(err) } + + ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil) + if !ch || err != nil { + t.Fatal(ch, err) + } + ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil) + if !ch || err != nil { + t.Fatal(ch, err) + } + ch, err = frame.SetFieldValue(1, "myfield", 87) + if !ch || err != nil { + t.Fatal(ch, err) + } + views := frame.Views() + if len(views) != 3 { + var names string + for _, v := range views { + names = names + v.Name() + " " + } + t.Fatalf("Unexpected views: %s", names) + } + }) t.Run("ErrRangeCacheAllowed", func(t *testing.T) { From 54680652137aee9b16cb4999b15d66d6c27020fa Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Apr 2018 13:01:36 -0500 Subject: [PATCH 35/79] improve apiFunc error handling. change slice to map in function validation. --- Makefile | 6 +- api.go | 173 +++++++++++++++++++++++++--------------------- apifunc_string.go | 16 +++++ pilosa.go | 6 ++ 4 files changed, 120 insertions(+), 81 deletions(-) create mode 100644 apifunc_string.go diff --git a/Makefile b/Makefile index 9e2c6fa3f..cab932fba 100644 --- a/Makefile +++ b/Makefile @@ -80,8 +80,12 @@ generate-protoc: require-protoc require-protoc-gen-gofast generate-statik: require-statik go generate github.com/pilosa/pilosa/statik +# `go generate` stringers +generate-stringer: + go generate github.com/pilosa/pilosa + # `go generate` all needed packages -generate: generate-protoc generate-statik +generate: generate-protoc generate-statik generate-stringer # Create Docker image from Dockerfile docker: diff --git a/api.go b/api.go index 970766842..6852a1f87 100644 --- a/api.go +++ b/api.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:generate stringer -type=apiFunc + package pilosa import ( @@ -61,26 +63,35 @@ func NewAPI() *API { // functionStates specifies the api functions that are valid for each // cluster state. -var functionStates = map[string][]int{ +var functionStates = map[string]map[apiFunc]struct{}{ ClusterStateStarting: functionCommon, - ClusterStateNormal: append(functionCommon, functionNormal...), - ClusterStateResizing: append(functionCommon, functionResizing...), + ClusterStateNormal: appendMap(functionCommon, functionNormal), + ClusterStateResizing: appendMap(functionCommon, functionResizing), } -func (api *API) validate(f int) error { - state := api.Cluster.State() - for _, fnc := range functionStates[state] { - if f == fnc { - return nil - } +func appendMap(a, b map[apiFunc]struct{}) map[apiFunc]struct{} { + r := make(map[apiFunc]struct{}) + for k, v := range a { + r[k] = v } - return fmt.Errorf("api function not allowed in state %s", state) + for k, v := range b { + r[k] = v + } + return r +} + +func (api *API) validate(f apiFunc) error { + state := api.Cluster.State() + if _, ok := functionStates[state][f]; ok { + return nil + } + return ApiFunctionNotAllowedError{errors.Errorf("api function %s not allowed in state %s", f, state)} } // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { if err := api.validate(apiQuery); err != nil { - return QueryResponse{}, errors.Wrap(err, "validate api function: query") + return QueryResponse{}, errors.Wrap(err, "validate api function") } resp := QueryResponse{} @@ -148,7 +159,7 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { if err := api.validate(apiCreateIndex); err != nil { - return nil, errors.Wrap(err, "validate api function: create index") + return nil, errors.Wrap(err, "validate api function") } // Create index. @@ -173,7 +184,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index // Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { if err := api.validate(apiIndex); err != nil { - return nil, errors.Wrap(err, "validate api function: index") + return nil, errors.Wrap(err, "validate api function") } index := api.Holder.Index(indexName) @@ -187,7 +198,7 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { // nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { if err := api.validate(apiDeleteIndex); err != nil { - return errors.Wrap(err, "validate api function: delete index") + return errors.Wrap(err, "validate api function") } // Delete index from the holder. @@ -211,7 +222,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { if err := api.validate(apiCreateFrame); err != nil { - return nil, errors.Wrap(err, "validate api function: create frame") + return nil, errors.Wrap(err, "validate api function") } // Find index. @@ -246,7 +257,7 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { if err := api.validate(apiDeleteFrame); err != nil { - return errors.Wrap(err, "validate api function: delete frame") + return errors.Wrap(err, "validate api function") } // Find index. @@ -278,7 +289,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { - return errors.Wrap(err, "validate api function: export csv") + return errors.Wrap(err, "validate api function") } // Validate that this handler owns the slice. @@ -315,7 +326,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin // SliceNodes returns the node and all replicas which should contain a slice's data. func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { if err := api.validate(apiSliceNodes); err != nil { - return nil, errors.Wrap(err, "validate api function: slice nodes") + return nil, errors.Wrap(err, "validate api function") } return api.Cluster.SliceNodes(indexName, slice), nil @@ -326,7 +337,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) // the UnmarshalFragment API call. func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { - return nil, errors.Wrap(err, "validate api function: marshal fragment") + return nil, errors.Wrap(err, "validate api function") } // Retrieve fragment from holder. @@ -342,7 +353,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // fragment's data. func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { - return errors.Wrap(err, "validate api function: unmarshal fragment") + return errors.Wrap(err, "validate api function") } // Retrieve frame. @@ -375,7 +386,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // ids from a "block" which is a subdivision of a fragment. func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { if err := api.validate(apiFragmentBlockData); err != nil { - return nil, errors.Wrap(err, "validate api function: fragment block data") + return nil, errors.Wrap(err, "validate api function") } reqBytes, err := ioutil.ReadAll(body) @@ -408,7 +419,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { - return nil, errors.Wrap(err, "validate api function: fragment blocks") + return nil, errors.Wrap(err, "validate api function") } // Retrieve fragment from holder. @@ -426,7 +437,7 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { if err := api.validate(apiRestoreFrame); err != nil { - return errors.Wrap(err, "validate api function: restore frame") + return errors.Wrap(err, "validate api function") } // Create a client for the remote cluster. @@ -504,7 +515,7 @@ func (api *API) Hosts(ctx context.Context) []*Node { // CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { if err := api.validate(apiCreateInputDefinition); err != nil { - return errors.Wrap(err, "validate api function: create input definition") + return errors.Wrap(err, "validate api function") } api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. @@ -542,7 +553,7 @@ Please open an issue if you need to continue using it.`) // InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { if err := api.validate(apiInputDefinition); err != nil { - return nil, errors.Wrap(err, "validate api function: input definition") + return nil, errors.Wrap(err, "validate api function") } api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) @@ -562,7 +573,7 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN // DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { if err := api.validate(apiDeleteInputDefinition); err != nil { - return errors.Wrap(err, "validate api function: delete input definition") + return errors.Wrap(err, "validate api function") } api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") @@ -591,7 +602,7 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp // WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { if err := api.validate(apiWriteInput); err != nil { - return errors.Wrap(err, "validate api function: write input") + return errors.Wrap(err, "validate api function") } api.Logger.Printf("WriteInput is deprecated and will be removed.") @@ -619,7 +630,7 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { if err := api.validate(apiRecalculateCaches); err != nil { - return errors.Wrap(err, "validate api function: recalculate caches") + return errors.Wrap(err, "validate api function") } err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) @@ -634,7 +645,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { // the body and forwards it to the BroadcastHandler. func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { if err := api.validate(apiClusterMessage); err != nil { - return errors.Wrap(err, "validate api function: cluster message") + return errors.Wrap(err, "validate api function") } // Read entire body. @@ -670,7 +681,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { if err := api.validate(apiCreateField); err != nil { - return errors.Wrap(err, "validate api function: create field") + return errors.Wrap(err, "validate api function") } // Retrieve frame by name. @@ -700,7 +711,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { if err := api.validate(apiDeleteField); err != nil { - return errors.Wrap(err, "validate api function: delete field") + return errors.Wrap(err, "validate api function") } // Retrieve frame by name. @@ -730,7 +741,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str // Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { if err := api.validate(apiFields); err != nil { - return nil, errors.Wrap(err, "validate api function: fields") + return nil, errors.Wrap(err, "validate api function") } index := api.Holder.index(indexName) @@ -749,7 +760,7 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { - return nil, errors.Wrap(err, "validate api function: views") + return nil, errors.Wrap(err, "validate api function") } // Retrieve views. @@ -766,7 +777,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { if err := api.validate(apiDeleteView); err != nil { - return errors.Wrap(err, "validate api function: delete view") + return errors.Wrap(err, "validate api function") } // Retrieve frame. @@ -800,7 +811,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiIndexAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function: index attr diff") + return nil, errors.Wrap(err, "validate api function") } // Retrieve index from holder. @@ -834,7 +845,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiFrameAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function: frame attr diff") + return nil, errors.Wrap(err, "validate api function") } // Retrieve index from holder. @@ -869,7 +880,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { - return errors.Wrap(err, "validate api function: import") + return errors.Wrap(err, "validate api function") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -898,7 +909,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { - return errors.Wrap(err, "validate api function: import value") + return errors.Wrap(err, "validate api function") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -917,7 +928,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyIndexTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function: modify index time quantum") + return errors.Wrap(err, "validate api function") } // Retrieve index by name. @@ -934,7 +945,7 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyFrameTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function: modify frame time quantum") + return errors.Wrap(err, "validate api function") } // Retrieve index by name. @@ -1072,7 +1083,7 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na // SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { if err := api.validate(apiSetCoordinator); err != nil { - return nil, nil, errors.Wrap(err, "validate api function: set coordinator") + return nil, nil, errors.Wrap(err, "validate api function") } oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) @@ -1102,7 +1113,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { if err := api.validate(apiRemoveNode); err != nil { - return nil, errors.Wrap(err, "validate api function: remove node") + return nil, errors.Wrap(err, "validate api function") } removeNode := api.Cluster.nodeByID(id) @@ -1121,7 +1132,7 @@ func (api *API) RemoveNode(id string) (*Node, error) { // ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { if err := api.validate(apiResizeAbort); err != nil { - return errors.Wrap(err, "validate api function: resize abort") + return errors.Wrap(err, "validate api function") } if !api.Cluster.IsCoordinator() { @@ -1143,9 +1154,11 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } +type apiFunc int + // API validation constants. const ( - apiClusterMessage int = iota + apiClusterMessage apiFunc = iota apiCreateField apiCreateFrame apiCreateIndex @@ -1189,44 +1202,44 @@ const ( apiWriteInput ) -var functionCommon = []int{ - apiClusterMessage, - apiMarshalFragment, - apiSetCoordinator, +var functionCommon = map[apiFunc]struct{}{ + apiClusterMessage: struct{}{}, + apiMarshalFragment: struct{}{}, + apiSetCoordinator: struct{}{}, } -var functionResizing = []int{ - apiResizeAbort, +var functionResizing = map[apiFunc]struct{}{ + apiResizeAbort: struct{}{}, } -var functionNormal = []int{ - apiCreateField, - apiCreateFrame, - apiCreateIndex, - apiCreateInputDefinition, - apiDeleteField, - apiDeleteFrame, - apiDeleteIndex, - apiDeleteInputDefinition, - apiDeleteView, - apiExportCSV, - apiFields, - apiFragmentBlockData, - apiFragmentBlocks, - apiFrameAttrDiff, - apiImport, - apiImportValue, - apiIndex, - apiIndexAttrDiff, - apiInputDefinition, - apiModifyFrameTimeQuantum, - apiModifyIndexTimeQuantum, - apiQuery, - apiRecalculateCaches, - apiRemoveNode, - apiRestoreFrame, - apiSliceNodes, - apiUnmarshalFragment, - apiViews, - apiWriteInput, +var functionNormal = map[apiFunc]struct{}{ + apiCreateField: struct{}{}, + apiCreateFrame: struct{}{}, + apiCreateIndex: struct{}{}, + apiCreateInputDefinition: struct{}{}, + apiDeleteField: struct{}{}, + apiDeleteFrame: struct{}{}, + apiDeleteIndex: struct{}{}, + apiDeleteInputDefinition: struct{}{}, + apiDeleteView: struct{}{}, + apiExportCSV: struct{}{}, + apiFields: struct{}{}, + apiFragmentBlockData: struct{}{}, + apiFragmentBlocks: struct{}{}, + apiFrameAttrDiff: struct{}{}, + apiImport: struct{}{}, + apiImportValue: struct{}{}, + apiIndex: struct{}{}, + apiIndexAttrDiff: struct{}{}, + apiInputDefinition: struct{}{}, + apiModifyFrameTimeQuantum: struct{}{}, + apiModifyIndexTimeQuantum: struct{}{}, + apiQuery: struct{}{}, + apiRecalculateCaches: struct{}{}, + apiRemoveNode: struct{}{}, + apiRestoreFrame: struct{}{}, + apiSliceNodes: struct{}{}, + apiUnmarshalFragment: struct{}{}, + apiViews: struct{}{}, + apiWriteInput: struct{}{}, } diff --git a/apifunc_string.go b/apifunc_string.go new file mode 100644 index 000000000..8fb9b16e0 --- /dev/null +++ b/apifunc_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type=apiFunc"; DO NOT EDIT. + +package pilosa + +import "fmt" + +const _apiFunc_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" + +var _apiFunc_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} + +func (i apiFunc) String() string { + if i < 0 || i >= apiFunc(len(_apiFunc_index)-1) { + return fmt.Sprintf("apiFunc(%d)", i) + } + return _apiFunc_name[_apiFunc_index[i]:_apiFunc_index[i+1]] +} diff --git a/pilosa.go b/pilosa.go index ffb836ae8..c3a6919db 100644 --- a/pilosa.go +++ b/pilosa.go @@ -82,6 +82,12 @@ var ( ErrResizeNotRunning = errors.New("no resize job currently running") ) +// InvalidApiFunctionError wraps an error value indicating that a particular +// API function is not allowed in the current cluster state. +type ApiFunctionNotAllowedError struct { + error +} + // BadRequestError wraps an error value to signify that a request could not be // read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest // would be returned. From e60d11a23e2d93ba5d594dac15f3d40a3d40eb0e Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Apr 2018 17:07:41 -0500 Subject: [PATCH 36/79] change references from "function" to "method" --- api.go | 98 ++++++++++++++++++++++----------------------- apifunc_string.go | 16 -------- apimethod_string.go | 16 ++++++++ pilosa.go | 6 +-- 4 files changed, 68 insertions(+), 68 deletions(-) delete mode 100644 apifunc_string.go create mode 100644 apimethod_string.go diff --git a/api.go b/api.go index 6852a1f87..ed41328bf 100644 --- a/api.go +++ b/api.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate stringer -type=apiFunc +//go:generate stringer -type=apiMethod package pilosa @@ -61,16 +61,16 @@ func NewAPI() *API { } } -// functionStates specifies the api functions that are valid for each +// validAPIMethods specifies the api methods that are valid for each // cluster state. -var functionStates = map[string]map[apiFunc]struct{}{ - ClusterStateStarting: functionCommon, - ClusterStateNormal: appendMap(functionCommon, functionNormal), - ClusterStateResizing: appendMap(functionCommon, functionResizing), +var validAPIMethods = map[string]map[apiMethod]struct{}{ + ClusterStateStarting: methodsCommon, + ClusterStateNormal: appendMap(methodsCommon, methodsNormal), + ClusterStateResizing: appendMap(methodsCommon, methodsResizing), } -func appendMap(a, b map[apiFunc]struct{}) map[apiFunc]struct{} { - r := make(map[apiFunc]struct{}) +func appendMap(a, b map[apiMethod]struct{}) map[apiMethod]struct{} { + r := make(map[apiMethod]struct{}) for k, v := range a { r[k] = v } @@ -80,18 +80,18 @@ func appendMap(a, b map[apiFunc]struct{}) map[apiFunc]struct{} { return r } -func (api *API) validate(f apiFunc) error { +func (api *API) validate(f apiMethod) error { state := api.Cluster.State() - if _, ok := functionStates[state][f]; ok { + if _, ok := validAPIMethods[state][f]; ok { return nil } - return ApiFunctionNotAllowedError{errors.Errorf("api function %s not allowed in state %s", f, state)} + return ApiMethodNotAllowedError{errors.Errorf("api method %s not allowed in state %s", f, state)} } // Query parses a PQL query out of the request and executes it. func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) { if err := api.validate(apiQuery); err != nil { - return QueryResponse{}, errors.Wrap(err, "validate api function") + return QueryResponse{}, errors.Wrap(err, "validate api method") } resp := QueryResponse{} @@ -159,7 +159,7 @@ func (api *API) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet // CreateIndex makes a new Pilosa index. func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error) { if err := api.validate(apiCreateIndex); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Create index. @@ -184,7 +184,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index // Index retrieves the named index. func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { if err := api.validate(apiIndex); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } index := api.Holder.Index(indexName) @@ -198,7 +198,7 @@ func (api *API) Index(ctx context.Context, indexName string) (*Index, error) { // nothing and returns no error. func (api *API) DeleteIndex(ctx context.Context, indexName string) error { if err := api.validate(apiDeleteIndex); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Delete index from the holder. @@ -222,7 +222,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { // CreateFrame makes the named frame in the named index with the given options. func (api *API) CreateFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) (*Frame, error) { if err := api.validate(apiCreateFrame); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Find index. @@ -257,7 +257,7 @@ func (api *API) CreateFrame(ctx context.Context, indexName string, frameName str // action is taken. func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName string) error { if err := api.validate(apiDeleteFrame); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Find index. @@ -289,7 +289,7 @@ func (api *API) DeleteFrame(ctx context.Context, indexName string, frameName str // CSV of the form , func (api *API) ExportCSV(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, w io.Writer) error { if err := api.validate(apiExportCSV); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Validate that this handler owns the slice. @@ -326,7 +326,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin // SliceNodes returns the node and all replicas which should contain a slice's data. func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) ([]*Node, error) { if err := api.validate(apiSliceNodes); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } return api.Cluster.SliceNodes(indexName, slice), nil @@ -337,7 +337,7 @@ func (api *API) SliceNodes(ctx context.Context, indexName string, slice uint64) // the UnmarshalFragment API call. func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) (io.WriterTo, error) { if err := api.validate(apiMarshalFragment); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve fragment from holder. @@ -353,7 +353,7 @@ func (api *API) MarshalFragment(ctx context.Context, indexName string, frameName // fragment's data. func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameName string, viewName string, slice uint64, reader io.ReadCloser) error { if err := api.validate(apiUnmarshalFragment); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame. @@ -386,7 +386,7 @@ func (api *API) UnmarshalFragment(ctx context.Context, indexName string, frameNa // ids from a "block" which is a subdivision of a fragment. func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error) { if err := api.validate(apiFragmentBlockData); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } reqBytes, err := ioutil.ReadAll(body) @@ -419,7 +419,7 @@ func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, // FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment. func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName string, viewName string, slice uint64) ([]FragmentBlock, error) { if err := api.validate(apiFragmentBlocks); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve fragment from holder. @@ -437,7 +437,7 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName // from replicas in the cluster and restores that data to it. func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { if err := api.validate(apiRestoreFrame); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Create a client for the remote cluster. @@ -515,7 +515,7 @@ func (api *API) Hosts(ctx context.Context) []*Node { // CreateInputDefinition is deprecated and will be removed. Do not use it. func (api *API) CreateInputDefinition(ctx context.Context, indexName string, inputDefName string, inputDef InputDefinitionInfo) error { if err := api.validate(apiCreateInputDefinition); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } api.Logger.Printf(`CreateInputDefinition is deprecated and will be removed. @@ -553,7 +553,7 @@ Please open an issue if you need to continue using it.`) // InputDefinition is deprecated and will be removed. func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefName string) (*InputDefinition, error) { if err := api.validate(apiInputDefinition); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } api.Logger.Printf(`InputDefinition is deprecated and will be removed.`) @@ -573,7 +573,7 @@ func (api *API) InputDefinition(ctx context.Context, indexName string, inputDefN // DeleteInputDefinition is deprecated and will be removed. func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inputDefName string) error { if err := api.validate(apiDeleteInputDefinition); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } api.Logger.Printf("DeleteInputDefinition is deprecated and will be removed.") @@ -602,7 +602,7 @@ func (api *API) DeleteInputDefinition(ctx context.Context, indexName string, inp // WriteInput is deprecated and will be removed. func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName string, reqs []interface{}) error { if err := api.validate(apiWriteInput); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } api.Logger.Printf("WriteInput is deprecated and will be removed.") @@ -630,7 +630,7 @@ func (api *API) WriteInput(ctx context.Context, indexName string, inputDefName s // RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests. func (api *API) RecalculateCaches(ctx context.Context) error { if err := api.validate(apiRecalculateCaches); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } err := api.Broadcaster.SendSync(&internal.RecalculateCaches{}) @@ -645,7 +645,7 @@ func (api *API) RecalculateCaches(ctx context.Context) error { // the body and forwards it to the BroadcastHandler. func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error { if err := api.validate(apiClusterMessage); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Read entire body. @@ -681,7 +681,7 @@ func (api *API) Schema(ctx context.Context) []*IndexInfo { // CreateField creates a new BSI field in the given index and frame. func (api *API) CreateField(ctx context.Context, indexName string, frameName string, field *Field) error { if err := api.validate(apiCreateField); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame by name. @@ -711,7 +711,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, frameName str // DeleteField deletes the given field. func (api *API) DeleteField(ctx context.Context, indexName string, frameName string, fieldName string) error { if err := api.validate(apiDeleteField); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame by name. @@ -741,7 +741,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, frameName str // Fields returns the fields in the given frame. func (api *API) Fields(ctx context.Context, indexName string, frameName string) ([]*Field, error) { if err := api.validate(apiFields); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } index := api.Holder.index(indexName) @@ -760,7 +760,7 @@ func (api *API) Fields(ctx context.Context, indexName string, frameName string) // Views returns the views in the given frame. func (api *API) Views(ctx context.Context, indexName string, frameName string) ([]*View, error) { if err := api.validate(apiViews); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve views. @@ -777,7 +777,7 @@ func (api *API) Views(ctx context.Context, indexName string, frameName string) ( // DeleteView removes the given view. func (api *API) DeleteView(ctx context.Context, indexName string, frameName string, viewName string) error { if err := api.validate(apiDeleteView); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve frame. @@ -811,7 +811,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // IndexAttrDiff func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiIndexAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve index from holder. @@ -845,7 +845,7 @@ func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []At func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error) { if err := api.validate(apiFrameAttrDiff); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } // Retrieve index from holder. @@ -880,7 +880,7 @@ func (api *API) FrameAttrDiff(ctx context.Context, indexName string, frameName s // Import bulk imports data into a particular index,frame,slice. func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { if err := api.validate(apiImport); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -909,7 +909,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // ImportValue bulk imports values into a particular field. func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest) error { if err := api.validate(apiImportValue); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } _, frame, err := api.indexFrame(req.Index, req.Frame, req.Slice) @@ -928,7 +928,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // ModifyIndexTimeQuantum changes the default time quantum on the given index. func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyIndexTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve index by name. @@ -945,7 +945,7 @@ func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, ti // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { if err := api.validate(apiModifyFrameTimeQuantum); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } // Retrieve index by name. @@ -1083,7 +1083,7 @@ func (api *API) inputJSONDataParser(req map[string]interface{}, index *Index, na // SetCoordinator makes a new Node the cluster coordinator. func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error) { if err := api.validate(apiSetCoordinator); err != nil { - return nil, nil, errors.Wrap(err, "validate api function") + return nil, nil, errors.Wrap(err, "validate api method") } oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) @@ -1113,7 +1113,7 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode // removing the given node. func (api *API) RemoveNode(id string) (*Node, error) { if err := api.validate(apiRemoveNode); err != nil { - return nil, errors.Wrap(err, "validate api function") + return nil, errors.Wrap(err, "validate api method") } removeNode := api.Cluster.nodeByID(id) @@ -1132,7 +1132,7 @@ func (api *API) RemoveNode(id string) (*Node, error) { // ResizeAbort stops the current resize job. func (api *API) ResizeAbort() error { if err := api.validate(apiResizeAbort); err != nil { - return errors.Wrap(err, "validate api function") + return errors.Wrap(err, "validate api method") } if !api.Cluster.IsCoordinator() { @@ -1154,11 +1154,11 @@ func (api *API) Version() string { return strings.TrimPrefix(Version, "v") } -type apiFunc int +type apiMethod int // API validation constants. const ( - apiClusterMessage apiFunc = iota + apiClusterMessage apiMethod = iota apiCreateField apiCreateFrame apiCreateIndex @@ -1202,17 +1202,17 @@ const ( apiWriteInput ) -var functionCommon = map[apiFunc]struct{}{ +var methodsCommon = map[apiMethod]struct{}{ apiClusterMessage: struct{}{}, apiMarshalFragment: struct{}{}, apiSetCoordinator: struct{}{}, } -var functionResizing = map[apiFunc]struct{}{ +var methodsResizing = map[apiMethod]struct{}{ apiResizeAbort: struct{}{}, } -var functionNormal = map[apiFunc]struct{}{ +var methodsNormal = map[apiMethod]struct{}{ apiCreateField: struct{}{}, apiCreateFrame: struct{}{}, apiCreateIndex: struct{}{}, diff --git a/apifunc_string.go b/apifunc_string.go deleted file mode 100644 index 8fb9b16e0..000000000 --- a/apifunc_string.go +++ /dev/null @@ -1,16 +0,0 @@ -// Code generated by "stringer -type=apiFunc"; DO NOT EDIT. - -package pilosa - -import "fmt" - -const _apiFunc_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" - -var _apiFunc_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} - -func (i apiFunc) String() string { - if i < 0 || i >= apiFunc(len(_apiFunc_index)-1) { - return fmt.Sprintf("apiFunc(%d)", i) - } - return _apiFunc_name[_apiFunc_index[i]:_apiFunc_index[i+1]] -} diff --git a/apimethod_string.go b/apimethod_string.go new file mode 100644 index 000000000..8a3dce195 --- /dev/null +++ b/apimethod_string.go @@ -0,0 +1,16 @@ +// Code generated by "stringer -type=apiMethod"; DO NOT EDIT. + +package pilosa + +import "fmt" + +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" + +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} + +func (i apiMethod) String() string { + if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { + return fmt.Sprintf("apiMethod(%d)", i) + } + return _apiMethod_name[_apiMethod_index[i]:_apiMethod_index[i+1]] +} diff --git a/pilosa.go b/pilosa.go index c3a6919db..347fbed67 100644 --- a/pilosa.go +++ b/pilosa.go @@ -82,9 +82,9 @@ var ( ErrResizeNotRunning = errors.New("no resize job currently running") ) -// InvalidApiFunctionError wraps an error value indicating that a particular -// API function is not allowed in the current cluster state. -type ApiFunctionNotAllowedError struct { +// ApiMethodNotAllowedError wraps an error value indicating that a particular +// API method is not allowed in the current cluster state. +type ApiMethodNotAllowedError struct { error } From e7151eb2a8a029ddf849b045c6d0070c3159c6b1 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 18 Apr 2018 08:17:26 -0500 Subject: [PATCH 37/79] Remove Index.TimeQuantum --- api.go | 18 ---- apimethod_string.go | 4 +- cmd/import.go | 1 - handler.go | 40 -------- handler_test.go | 20 ---- holder.go | 1 - holder_test.go | 16 ---- index.go | 63 +++--------- index_test.go | 50 +--------- internal/private.pb.go | 213 ++++++++++++++++------------------------- internal/private.proto | 1 - server.go | 4 +- 12 files changed, 98 insertions(+), 333 deletions(-) diff --git a/api.go b/api.go index ed41328bf..a5953217d 100644 --- a/api.go +++ b/api.go @@ -925,22 +925,6 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return err } -// ModifyIndexTimeQuantum changes the default time quantum on the given index. -func (api *API) ModifyIndexTimeQuantum(ctx context.Context, indexName string, timeQuantum TimeQuantum) error { - if err := api.validate(apiModifyIndexTimeQuantum); err != nil { - return errors.Wrap(err, "validate api method") - } - - // Retrieve index by name. - index := api.Holder.Index(indexName) - if index == nil { - return ErrIndexNotFound - } - - // Set default time quantum on index. - return index.SetTimeQuantum(timeQuantum) -} - // ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO: // what happens if there is already data in the frame? func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { @@ -1185,7 +1169,6 @@ const ( //apiMaxInverseSlices // not implemented //apiMaxSlices // not implemented apiModifyFrameTimeQuantum - apiModifyIndexTimeQuantum apiQuery apiRecalculateCaches apiRemoveNode @@ -1233,7 +1216,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiIndexAttrDiff: struct{}{}, apiInputDefinition: struct{}{}, apiModifyFrameTimeQuantum: struct{}{}, - apiModifyIndexTimeQuantum: struct{}{}, apiQuery: struct{}{}, apiRecalculateCaches: struct{}{}, apiRemoveNode: struct{}{}, diff --git a/apimethod_string.go b/apimethod_string.go index 8a3dce195..8f77b69e7 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,9 @@ package pilosa import "fmt" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiModifyIndexTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 369, 377, 397, 410, 424, 439, 456, 469, 489, 497, 510} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 352, 372, 385, 399, 414, 431, 444, 464, 472, 485} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/cmd/import.go b/cmd/import.go index b6b78bdbc..93fa92ee0 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -60,7 +60,6 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") - flags.Var(&Importer.IndexOptions.TimeQuantum, "index-time-quantum", "Time quantum for the index (DEPRECATED. This feature will be removed in a future version. Set time quantum of each frame instead.)") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") flags.BoolVar(&Importer.FrameOptions.InverseEnabled, "frame-inverse-enabled", false, "Enable inverse frame") flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.") diff --git a/handler.go b/handler.go index aa630e52c..bd97c1d1a 100644 --- a/handler.go +++ b/handler.go @@ -168,7 +168,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handlePostInputDefinition).Methods("POST") router.HandleFunc("/index/{index}/input-definition/{input-definition}", handler.handleDeleteInputDefinition).Methods("DELETE") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") - router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") // TODO: Apply MethodNotAllowed statuses to all endpoints. @@ -457,45 +456,6 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { } } -// handlePatchIndexTimeQuantum handles PATCH /index/time_quantum request. -func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - - // Decode request. - var req patchIndexTimeQuantumRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Validate quantum. - tq, err := ParseTimeQuantum(req.TimeQuantum) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err = h.API.ModifyIndexTimeQuantum(r.Context(), indexName, tq); err != nil { - if err == ErrIndexNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(patchIndexTimeQuantumResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type patchIndexTimeQuantumRequest struct { - TimeQuantum string `json:"timeQuantum"` -} - -type patchIndexTimeQuantumResponse struct{} - // handlePostIndexAttrDiff handles POST /index/attr/diff requests. func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] diff --git a/handler_test.go b/handler_test.go index c9044aee0..8c719feb1 100644 --- a/handler_test.go +++ b/handler_test.go @@ -748,26 +748,6 @@ func TestHandler_DeleteFrame(t *testing.T) { } } -// Ensure handler can set the Index time quantum. -func TestHandler_SetIndexTimeQuantum(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) - - h := test.NewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if q := hldr.Index("i0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected time quantum: %s", q) - } -} - // Ensure handler can set the frame time quantum. func TestHandler_SetFrameTimeQuantum(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/holder.go b/holder.go index 570433ce4..fa93767a3 100644 --- a/holder.go +++ b/holder.go @@ -359,7 +359,6 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { } // Update options. - index.SetTimeQuantum(opt.TimeQuantum) h.indexes[index.Name()] = index diff --git a/holder_test.go b/holder_test.go index 24594025d..8b1f6166a 100644 --- a/holder_test.go +++ b/holder_test.go @@ -73,22 +73,6 @@ func TestHolder_Open(t *testing.T) { t.Fatalf("unexpected error: %s", err) } }) - t.Run("ErrIndexMetaCorrupt", func(t *testing.T) { - h := test.MustOpenHolder() - defer h.Close() - - if _, err := h.CreateIndex("test", pilosa.IndexOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}); err != nil { - t.Fatal(err) - } else if err := h.Holder.Close(); err != nil { - t.Fatal(err) - } else if err := os.Truncate(filepath.Join(h.IndexPath("test"), ".meta"), 2); err != nil { - t.Fatal(err) - } - - if err := h.Reopen(); err == nil || !strings.Contains(err.Error(), "unexpected EOF") { - t.Fatalf("unexpected error: %s", err) - } - }) t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) { h := test.MustOpenHolder() defer h.Close() diff --git a/index.go b/index.go index 039816322..4676e1b43 100644 --- a/index.go +++ b/index.go @@ -39,10 +39,6 @@ type Index struct { path string name string - // Default time quantum for all frames in index. - // This can be overridden by individual frames. - timeQuantum TimeQuantum - // Frames by name. frames map[string]*Frame @@ -106,9 +102,7 @@ func (i *Index) Options() IndexOptions { } func (i *Index) options() IndexOptions { - return IndexOptions{ - TimeQuantum: i.timeQuantum, - } + return IndexOptions{} } // Open opens and initializes the index. @@ -175,7 +169,6 @@ func (i *Index) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta")) if os.IsNotExist(err) { - i.timeQuantum = "" return nil } else if err != nil { return err @@ -186,17 +179,18 @@ func (i *Index) loadMeta() error { } // Copy metadata fields. - i.timeQuantum = TimeQuantum(pb.TimeQuantum) return nil } +// NOTE: Until we introduce new attributes to store in the index .meta file, +// we don't need to actually write the file. The code related to index.options +// and the index meta file are left in place for future use. +/* // saveMeta writes meta data for the index. func (i *Index) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.IndexMeta{ - TimeQuantum: string(i.timeQuantum), - }) + buf, err := proto.Marshal(&internal.IndexMeta{}) if err != nil { return err } @@ -208,6 +202,7 @@ func (i *Index) saveMeta() error { return nil } +*/ // Close closes the index and its frames. func (i *Index) Close() error { @@ -278,34 +273,6 @@ func (i *Index) SetRemoteMaxInverseSlice(v uint64) { i.remoteMaxInverseSlice = v } -// TimeQuantum returns the default time quantum for the index. -func (i *Index) TimeQuantum() TimeQuantum { - i.mu.RLock() - defer i.mu.RUnlock() - return i.timeQuantum -} - -// SetTimeQuantum sets the default time quantum for the index. -func (i *Index) SetTimeQuantum(q TimeQuantum) error { - i.mu.Lock() - defer i.mu.Unlock() - - // Validate input. - if !q.Valid() { - return ErrInvalidTimeQuantum - } - - // Update value on index. - i.timeQuantum = q - - // Perist meta data to disk. - if err := i.saveMeta(); err != nil { - return err - } - - return nil -} - // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } @@ -425,12 +392,8 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, err } - // Default the time quantum to what is set on the Index. - timeQuantum := i.timeQuantum - if opt.TimeQuantum != "" { - timeQuantum = opt.TimeQuantum - } - if err := f.SetTimeQuantum(timeQuantum); err != nil { + // Set the time quantum. + if err := f.SetTimeQuantum(opt.TimeQuantum); err != nil { f.Close() return nil, err } @@ -577,15 +540,11 @@ func encodeIndex(d *Index) *internal.Index { } // IndexOptions represents options to set when initializing an index. -type IndexOptions struct { - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` -} +type IndexOptions struct{} // Encode converts i into its internal representation. func (i *IndexOptions) Encode() *internal.IndexMeta { - return &internal.IndexMeta{ - TimeQuantum: string(i.TimeQuantum), - } + return &internal.IndexMeta{} } // hasTime returns true if a contains a non-nil time. diff --git a/index_test.go b/index_test.go index 25e093d4b..b0a5b3506 100644 --- a/index_test.go +++ b/index_test.go @@ -58,11 +58,6 @@ func TestIndex_CreateFrame(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - // Create frame with explicit quantum. f, err := index.CreateFrame("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YMDH")}) if err != nil { @@ -71,24 +66,6 @@ func TestIndex_CreateFrame(t *testing.T) { t.Fatalf("unexpected frame time quantum: %s", q) } }) - - t.Run("Inherited", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - - // Create frame. - f, err := index.CreateFrame("f", pilosa.FrameOptions{}) - if err != nil { - t.Fatal(err) - } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") { - t.Fatalf("unexpected frame time quantum: %s", q) - } - }) }) // Ensure frame can include range columns. @@ -267,26 +244,6 @@ func TestIndex_DeleteFrame(t *testing.T) { } } -// Ensure index can set the default time quantum. -func TestIndex_SetTimeQuantum(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - // Set & retrieve time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { - t.Fatal(err) - } else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum: %s", q) - } - - // Reload index and verify that it is persisted. - if err := index.Reopen(); err != nil { - t.Fatal(err) - } else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected quantum (reopen): %s", q) - } -} - // Ensure index can delete a frame. func TestIndex_InvalidName(t *testing.T) { path, err := ioutil.TempDir("", "pilosa-index-") @@ -404,18 +361,13 @@ func TestIndex_InputBits(t *testing.T) { index := test.MustOpenIndex() defer index.Close() - // Set index time quantum. - if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil { - t.Fatal(err) - } - err := index.InputBits("f", bits) if !strings.Contains(err.Error(), "Frame not found") { t.Fatalf("Expected Frame not found error, actual error: %s", err) } // Create frame. - if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{TimeQuantum: pilosa.TimeQuantum("YM")}); err != nil { t.Fatal(err) } diff --git a/internal/private.pb.go b/internal/private.pb.go index 0d0f3124a..2e4d39239 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -68,7 +68,6 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type IndexMeta struct { - TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *IndexMeta) Reset() { *m = IndexMeta{} } @@ -76,13 +75,6 @@ func (m *IndexMeta) String() string { return proto.CompactTextString( func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } -func (m *IndexMeta) GetTimeQuantum() string { - if m != nil { - return m.TimeQuantum - } - return "" -} - type FrameMeta struct { InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` @@ -1232,12 +1224,6 @@ func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.TimeQuantum) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) - i += copy(dAtA[i:], m.TimeQuantum) - } return i, nil } @@ -2746,10 +2732,6 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { func (m *IndexMeta) Size() (n int) { var l int _ = l - l = len(m.TimeQuantum) - if l > 0 { - n += 1 + l + sovPrivate(uint64(l)) - } return n } @@ -3439,35 +3421,6 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.TimeQuantum = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8573,87 +8526,87 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1308 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xbd, 0xb6, 0x13, 0x3f, 0xd7, 0xa9, 0x33, 0x6d, 0x83, 0x5b, 0x45, 0xae, 0x19, 0x15, - 0x1a, 0x2a, 0x35, 0x2a, 0xa9, 0x84, 0x68, 0xa1, 0x52, 0x69, 0xec, 0xaa, 0x0b, 0xa4, 0x2a, 0xe3, - 0xb6, 0x48, 0x48, 0x20, 0x4d, 0xed, 0x21, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0x49, 0xdc, 0x03, - 0x47, 0x84, 0x84, 0xb8, 0x23, 0xae, 0x7c, 0x19, 0x8e, 0x7c, 0x02, 0x84, 0xc2, 0x87, 0xe0, 0x08, - 0x9a, 0x37, 0x33, 0xbb, 0xeb, 0x7f, 0x49, 0x13, 0xb8, 0xed, 0xfb, 0xff, 0x9b, 0xf7, 0x6f, 0x66, - 0xa1, 0x36, 0x8c, 0xfd, 0x7d, 0x2e, 0xc5, 0xe6, 0x30, 0x8e, 0x64, 0x44, 0x96, 0xfd, 0x50, 0x8a, - 0x38, 0xe4, 0x01, 0xbd, 0x09, 0x15, 0x2f, 0xec, 0x8b, 0xc3, 0x1d, 0x21, 0x39, 0x69, 0x41, 0xf5, - 0xa9, 0x3f, 0x10, 0x9f, 0x8f, 0x78, 0x28, 0x47, 0x83, 0x46, 0xa1, 0xe5, 0x6c, 0x54, 0x58, 0x9e, - 0x45, 0xff, 0x70, 0xa0, 0xf2, 0x30, 0xe6, 0x03, 0x81, 0xfa, 0xef, 0xc0, 0x8a, 0x17, 0xee, 0x8b, - 0x38, 0x11, 0x9d, 0x90, 0xbf, 0x08, 0x44, 0x1f, 0x4d, 0x96, 0xd9, 0x14, 0x97, 0xac, 0x43, 0x65, - 0x9b, 0xf7, 0x5e, 0x8a, 0xa7, 0xe3, 0xa1, 0x68, 0xb8, 0xe8, 0x35, 0x63, 0xa4, 0xd2, 0xae, 0xff, - 0x4a, 0x34, 0x8a, 0x2d, 0x67, 0xa3, 0xc6, 0x32, 0xc6, 0x34, 0xa6, 0xd2, 0x0c, 0x26, 0x42, 0xe1, - 0x1c, 0xe3, 0xe1, 0x6e, 0x8a, 0xa1, 0x8c, 0x18, 0x26, 0x78, 0xe4, 0x3a, 0x94, 0x1f, 0xfa, 0x22, - 0xe8, 0x27, 0x8d, 0xa5, 0x96, 0xbb, 0x51, 0xdd, 0x3a, 0xbf, 0x69, 0x33, 0xb0, 0x89, 0x7c, 0x66, - 0xc4, 0x94, 0xc2, 0x8a, 0x37, 0x18, 0x46, 0xb1, 0x64, 0x22, 0x19, 0x46, 0x61, 0x22, 0x48, 0x1d, - 0xdc, 0x4e, 0x1c, 0x37, 0x1c, 0x0c, 0xac, 0x3e, 0xe9, 0x77, 0x50, 0x7f, 0x10, 0x44, 0xbd, 0xbd, - 0x36, 0x97, 0x9c, 0x89, 0x6f, 0x47, 0x22, 0x91, 0xe4, 0x22, 0x94, 0x30, 0x8f, 0x46, 0x4f, 0x13, - 0x8a, 0x8b, 0xd9, 0x32, 0xa9, 0xd4, 0x84, 0xe2, 0xa2, 0x3d, 0xa6, 0xa2, 0xc8, 0x34, 0xa1, 0xb8, - 0xdd, 0xc0, 0xef, 0xe9, 0x14, 0x14, 0x99, 0x26, 0x08, 0x81, 0xe2, 0x73, 0x5f, 0x1c, 0x98, 0x73, - 0xe3, 0x37, 0xf5, 0x60, 0x35, 0x17, 0xdf, 0xc0, 0x5c, 0x83, 0x32, 0x8b, 0x0e, 0xbc, 0x76, 0xd2, - 0x70, 0x5a, 0xee, 0x46, 0x91, 0x19, 0x0a, 0xb3, 0x1b, 0x05, 0xa3, 0x41, 0xa8, 0x44, 0x05, 0x14, - 0x65, 0x0c, 0x7a, 0x19, 0x4a, 0x98, 0x6a, 0x75, 0xca, 0xcc, 0x56, 0x7d, 0xd2, 0x7f, 0x1c, 0xa8, - 0xec, 0xf0, 0x43, 0x84, 0x91, 0x90, 0x7b, 0xb0, 0xdc, 0x95, 0x3c, 0xec, 0xf3, 0xb8, 0x8f, 0x4a, - 0xd5, 0xad, 0xb7, 0xb2, 0x14, 0xa6, 0x6a, 0x9b, 0x56, 0xa7, 0x13, 0xca, 0x78, 0xcc, 0x52, 0x13, - 0x72, 0x17, 0x96, 0x4c, 0x4f, 0x20, 0x86, 0xea, 0x56, 0x6b, 0x9e, 0x75, 0xda, 0x36, 0xca, 0xd8, - 0x1a, 0x5c, 0xf9, 0x10, 0x6a, 0x13, 0x6e, 0x15, 0xd6, 0x3d, 0x31, 0xb6, 0x15, 0xd9, 0x13, 0x63, - 0x95, 0xbb, 0x7d, 0x1e, 0x8c, 0x74, 0x9e, 0x8b, 0x4c, 0x13, 0x77, 0x0b, 0x1f, 0x38, 0x57, 0xee, - 0xc2, 0xb9, 0xbc, 0xd7, 0xd3, 0xd8, 0xd2, 0xaf, 0x81, 0x6c, 0xc7, 0x82, 0x4b, 0x81, 0xf0, 0x76, - 0x44, 0x92, 0xf0, 0x5d, 0xb1, 0xb8, 0xd2, 0xba, 0x7a, 0x85, 0x7c, 0xf5, 0xd6, 0xa1, 0xe2, 0x25, - 0xf6, 0xe0, 0x2e, 0xf6, 0x65, 0xc6, 0xa0, 0x37, 0x80, 0xb4, 0x45, 0x20, 0xa4, 0x30, 0x13, 0x78, - 0x8c, 0x7f, 0xda, 0xb5, 0x58, 0x4e, 0xd6, 0x25, 0xd7, 0xa1, 0xa8, 0xc6, 0x13, 0xa1, 0x54, 0xb7, - 0x2e, 0x64, 0x99, 0x4e, 0x27, 0x9d, 0xa1, 0x02, 0xf5, 0xad, 0x53, 0x33, 0xd2, 0x27, 0x1c, 0x70, - 0x4e, 0x2b, 0xdb, 0x50, 0xee, 0x74, 0xa8, 0x74, 0x49, 0x98, 0x50, 0xf7, 0xed, 0x59, 0xcf, 0x1a, - 0x8a, 0xee, 0xa6, 0x60, 0xd5, 0xa4, 0x9e, 0x05, 0xec, 0xdb, 0x50, 0x42, 0x5b, 0x83, 0x76, 0x66, - 0x07, 0x68, 0x29, 0x7d, 0x9e, 0x42, 0x3d, 0x6b, 0xa0, 0x8b, 0xf9, 0x40, 0x15, 0xeb, 0xf7, 0x4b, - 0xa3, 0xab, 0x66, 0xfa, 0xb1, 0xb2, 0xd1, 0x9e, 0xf0, 0x7b, 0x71, 0xcd, 0xa6, 0x12, 0xa9, 0x7c, - 0xab, 0x25, 0x90, 0x34, 0xdc, 0x96, 0xab, 0x7c, 0x23, 0x41, 0x6f, 0x43, 0xb9, 0xdb, 0x7b, 0x29, - 0x06, 0x9c, 0xbc, 0xab, 0x26, 0xad, 0x2f, 0x0e, 0x45, 0x62, 0xe6, 0xf4, 0xfc, 0x54, 0xfd, 0x99, - 0x95, 0xd3, 0x1f, 0x1d, 0x73, 0xa6, 0x05, 0x88, 0xca, 0x18, 0x3b, 0x69, 0x14, 0x67, 0x56, 0xa6, - 0xe2, 0x33, 0x23, 0x26, 0x1d, 0xa8, 0x7b, 0xe1, 0x70, 0x24, 0xdb, 0xe2, 0x1b, 0x3f, 0xf4, 0xa5, - 0x1f, 0x85, 0x49, 0xa3, 0x8c, 0x26, 0x97, 0xf3, 0xa1, 0x27, 0x34, 0xd8, 0x8c, 0x09, 0xfd, 0xde, - 0x81, 0xf3, 0x53, 0xcc, 0x13, 0x70, 0x15, 0x8e, 0xc7, 0xf5, 0x7e, 0xba, 0xf3, 0x5d, 0x54, 0x6c, - 0x2e, 0x44, 0x33, 0x79, 0x05, 0xfc, 0xea, 0xc0, 0xc5, 0x79, 0x0a, 0x73, 0xd1, 0x34, 0x01, 0x9e, - 0xc4, 0xfe, 0x80, 0xc7, 0xe3, 0x4f, 0xc5, 0xd8, 0x5c, 0x7f, 0x39, 0x0e, 0xf9, 0x02, 0xd6, 0xa6, - 0x7c, 0x7d, 0xdc, 0xd3, 0x29, 0xd2, 0xa0, 0xae, 0x2e, 0x04, 0xa5, 0xf5, 0xd8, 0x02, 0x73, 0xfa, - 0xb7, 0x03, 0x97, 0xe6, 0x8a, 0xb2, 0x9e, 0x74, 0xf2, 0x3d, 0x79, 0x03, 0xea, 0xcf, 0xd5, 0x66, - 0x6b, 0x8b, 0x44, 0xfa, 0x21, 0x57, 0x9a, 0xa6, 0x69, 0x67, 0xf8, 0xc4, 0x83, 0x65, 0xe4, 0xed, - 0xf0, 0xa1, 0x81, 0x79, 0xf3, 0x04, 0x98, 0x9b, 0x56, 0xdf, 0x2c, 0x7e, 0x4b, 0x2a, 0x30, 0x78, - 0x11, 0xd9, 0x5b, 0x0d, 0x09, 0xb5, 0xd2, 0x27, 0x0c, 0x4e, 0xb5, 0x96, 0x23, 0x58, 0xb7, 0xab, - 0x70, 0x02, 0xc9, 0xf1, 0x93, 0x7a, 0x07, 0x20, 0x53, 0x35, 0x1b, 0xe0, 0x98, 0xfe, 0xcc, 0x29, - 0xd3, 0x47, 0xb0, 0x6e, 0xf7, 0xf4, 0x29, 0x02, 0xda, 0x6e, 0x29, 0x64, 0xdd, 0x42, 0x3b, 0xe0, - 0x3e, 0x63, 0x9e, 0xba, 0xab, 0x71, 0x5a, 0x6d, 0x89, 0x0c, 0xa5, 0x4c, 0x1e, 0x45, 0x89, 0xb4, - 0x26, 0xea, 0x5b, 0xf1, 0x9e, 0x44, 0xb1, 0x44, 0xc4, 0x35, 0x86, 0xdf, 0xf4, 0x2b, 0x28, 0x3e, - 0x8e, 0xfa, 0x82, 0xac, 0x40, 0xc1, 0x6b, 0x1b, 0x1f, 0x05, 0xaf, 0x4d, 0xae, 0xa2, 0x7b, 0xb3, - 0x43, 0x6a, 0xd9, 0xe1, 0x9e, 0x31, 0x8f, 0x61, 0xe0, 0x6b, 0x50, 0xf3, 0x92, 0xed, 0x28, 0x8a, - 0xfb, 0xaa, 0xd4, 0x51, 0x6c, 0xee, 0xa4, 0x49, 0x26, 0xbd, 0x0f, 0x75, 0xe5, 0xbe, 0x2b, 0xb9, - 0x4c, 0x37, 0xf5, 0x1a, 0x94, 0x15, 0x2f, 0x0d, 0x67, 0x28, 0xbc, 0xf7, 0x94, 0x9e, 0x5d, 0x80, - 0x48, 0xd0, 0xcf, 0xb4, 0x87, 0xce, 0xbe, 0x08, 0x65, 0x2e, 0x4b, 0x48, 0xa3, 0x83, 0x1a, 0xd3, - 0x04, 0xa1, 0xfa, 0x28, 0x06, 0xf3, 0x4a, 0x86, 0x59, 0x71, 0x19, 0xca, 0xe8, 0x4f, 0x0e, 0x80, - 0x05, 0x34, 0x4a, 0x52, 0x13, 0x67, 0xb1, 0x09, 0x79, 0x2f, 0xf7, 0x76, 0x99, 0xdd, 0xa9, 0xa9, - 0x88, 0xe5, 0x5e, 0x38, 0x1b, 0x76, 0x85, 0x9a, 0xe6, 0xa8, 0x67, 0xfa, 0x9a, 0x6f, 0xca, 0xa4, - 0xae, 0xcd, 0xda, 0x76, 0x30, 0x4a, 0xa4, 0x88, 0x0d, 0x22, 0xf5, 0xc6, 0xd2, 0x8c, 0x34, 0x3f, - 0x19, 0x63, 0x7e, 0x8a, 0xc8, 0x35, 0x28, 0x29, 0xa4, 0x76, 0x0f, 0x4c, 0x1f, 0x43, 0x0b, 0x69, - 0xd7, 0xdc, 0x24, 0x73, 0x77, 0x0f, 0x81, 0x22, 0xbe, 0xa8, 0x4d, 0xbb, 0xe0, 0x63, 0xba, 0x0e, - 0xee, 0x8e, 0xaf, 0xfb, 0xdb, 0x65, 0xea, 0x13, 0x39, 0xfc, 0x10, 0xe7, 0x4f, 0x71, 0xb8, 0x7a, - 0x4b, 0xac, 0xea, 0x01, 0x52, 0x77, 0xc7, 0x59, 0xee, 0x37, 0xfb, 0x28, 0x75, 0x73, 0x8f, 0xd2, - 0x2e, 0xac, 0xea, 0x21, 0xf9, 0x3f, 0x9d, 0xfe, 0x52, 0x80, 0x55, 0x26, 0x12, 0xff, 0x95, 0xf0, - 0xc2, 0x44, 0xc6, 0xa3, 0x74, 0xc1, 0x7d, 0x12, 0xbd, 0x30, 0xa9, 0x76, 0x99, 0x26, 0x5e, 0xa7, - 0x93, 0xc8, 0x2d, 0xa8, 0x4e, 0x77, 0xff, 0xac, 0x6a, 0x5e, 0x85, 0xdc, 0x82, 0xa5, 0x6e, 0x34, - 0x8a, 0x7b, 0xe9, 0x35, 0xb8, 0x96, 0x69, 0x6b, 0x64, 0x5a, 0xcc, 0xac, 0x5a, 0xae, 0x8f, 0x4a, - 0xc7, 0xf7, 0x11, 0xb9, 0x37, 0xd5, 0x47, 0xf8, 0xe7, 0x52, 0xdd, 0x7a, 0x33, 0x33, 0x98, 0x10, - 0xb3, 0x49, 0x6d, 0xfa, 0x83, 0x03, 0xe7, 0xf2, 0x10, 0x5e, 0x6b, 0x30, 0xd2, 0x8a, 0x14, 0xe6, - 0x56, 0xc4, 0x9d, 0x57, 0x91, 0x62, 0x56, 0x91, 0xec, 0x9d, 0x5b, 0xca, 0xbd, 0x73, 0xe9, 0x1e, - 0x5c, 0x9e, 0x29, 0xd3, 0x76, 0x34, 0x18, 0xaa, 0x7e, 0xf8, 0x0f, 0xe5, 0x52, 0x2b, 0x23, 0x8e, - 0x4d, 0xa1, 0x2a, 0x4c, 0x13, 0xf4, 0x0e, 0x5c, 0xea, 0x0a, 0x99, 0x2b, 0x92, 0xed, 0xb6, 0x16, - 0xb8, 0x8f, 0xc5, 0xc1, 0x82, 0xe3, 0x2b, 0x11, 0xfd, 0x08, 0x1a, 0xcf, 0x86, 0x7d, 0x2e, 0xc5, - 0x99, 0xac, 0x1f, 0xc0, 0xf2, 0xd3, 0x68, 0x18, 0x05, 0xd1, 0xee, 0xf8, 0x84, 0x91, 0x6f, 0xc0, - 0x92, 0xde, 0x8f, 0xfa, 0x91, 0x52, 0x61, 0x96, 0xa4, 0x17, 0x54, 0x43, 0xf7, 0x78, 0xd0, 0x1b, - 0x05, 0x0a, 0x86, 0xfa, 0xf7, 0x4a, 0x1e, 0xd4, 0x7f, 0x3b, 0x6a, 0x3a, 0xbf, 0x1f, 0x35, 0x9d, - 0x3f, 0x8f, 0x9a, 0xce, 0xcf, 0x7f, 0x35, 0xdf, 0x78, 0x51, 0xc6, 0xff, 0xf4, 0xdb, 0xff, 0x06, - 0x00, 0x00, 0xff, 0xff, 0xda, 0x68, 0xc4, 0x54, 0xb8, 0x0f, 0x00, 0x00, + // 1307 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4d, 0x6f, 0x1b, 0xc5, + 0x1b, 0xff, 0xaf, 0xd7, 0x76, 0xe2, 0xc7, 0x71, 0xe2, 0x4c, 0xd3, 0xfc, 0x37, 0x55, 0xe4, 0x9a, + 0x51, 0xa1, 0xa6, 0x12, 0x51, 0x49, 0x25, 0x44, 0x03, 0x95, 0x4a, 0x62, 0x57, 0x5d, 0x20, 0x51, + 0x19, 0x27, 0x41, 0x42, 0x02, 0x69, 0x62, 0x0f, 0xe9, 0x2a, 0xeb, 0x5d, 0xb3, 0x3b, 0x4e, 0xe2, + 0x1e, 0x38, 0x22, 0x24, 0xc4, 0x1d, 0x71, 0xe5, 0xcb, 0x70, 0xe4, 0x13, 0x20, 0x14, 0x3e, 0x04, + 0x47, 0xd0, 0xbc, 0xed, 0xae, 0xdf, 0x92, 0x26, 0x70, 0xdb, 0xe7, 0xfd, 0x37, 0xcf, 0xdb, 0xcc, + 0x42, 0xa5, 0x1f, 0x79, 0xa7, 0x94, 0xb3, 0x8d, 0x7e, 0x14, 0xf2, 0x10, 0xcd, 0x7b, 0x01, 0x67, + 0x51, 0x40, 0x7d, 0x5c, 0x86, 0x92, 0x1b, 0x74, 0xd9, 0xf9, 0x2e, 0xe3, 0x14, 0xff, 0x6e, 0x41, + 0xe9, 0x59, 0x44, 0x7b, 0x4c, 0x50, 0xe8, 0x2d, 0x58, 0x74, 0x83, 0x53, 0x16, 0xc5, 0xac, 0x15, + 0xd0, 0x23, 0x9f, 0x75, 0x9d, 0x5c, 0xdd, 0x6a, 0xcc, 0x93, 0x31, 0x2e, 0x5a, 0x87, 0xd2, 0x0e, + 0xed, 0xbc, 0x64, 0xfb, 0xc3, 0x3e, 0x73, 0xec, 0xba, 0xd5, 0x28, 0x91, 0x94, 0x91, 0x48, 0xdb, + 0xde, 0x2b, 0xe6, 0xe4, 0xeb, 0x56, 0xa3, 0x42, 0x52, 0x06, 0xaa, 0x43, 0x79, 0xdf, 0xeb, 0xb1, + 0xcf, 0x06, 0x34, 0xe0, 0x83, 0x9e, 0x53, 0x90, 0xd6, 0x59, 0x16, 0xc2, 0xb0, 0x40, 0x68, 0x70, + 0x9c, 0x60, 0x28, 0x4a, 0x0c, 0x23, 0x3c, 0x74, 0x1f, 0x8a, 0xcf, 0x3c, 0xe6, 0x77, 0x63, 0x67, + 0xae, 0x6e, 0x37, 0xca, 0x9b, 0x4b, 0x1b, 0xe6, 0x7c, 0x1b, 0x92, 0x4f, 0xb4, 0x18, 0x63, 0x58, + 0x74, 0x7b, 0xfd, 0x30, 0xe2, 0x84, 0xc5, 0xfd, 0x30, 0x88, 0x19, 0xaa, 0x82, 0xdd, 0x8a, 0x22, + 0xc7, 0x92, 0x81, 0xc5, 0x27, 0xfe, 0x16, 0xaa, 0xdb, 0x7e, 0xd8, 0x39, 0x69, 0x52, 0x4e, 0x09, + 0xfb, 0x66, 0xc0, 0x62, 0x8e, 0x56, 0xa0, 0x20, 0xb3, 0xa4, 0xf5, 0x14, 0x21, 0xb8, 0x32, 0x5b, + 0x32, 0x2f, 0x25, 0xa2, 0x08, 0xc1, 0x95, 0xf6, 0x32, 0x15, 0x79, 0xa2, 0x08, 0xc1, 0x6d, 0xfb, + 0x5e, 0x47, 0xa5, 0x20, 0x4f, 0x14, 0x81, 0x10, 0xe4, 0x0f, 0x3d, 0x76, 0xa6, 0xcf, 0x2d, 0xbf, + 0xb1, 0x0b, 0xcb, 0x99, 0xf8, 0x1a, 0xe6, 0x2a, 0x14, 0x49, 0x78, 0xe6, 0x36, 0x63, 0xc7, 0xaa, + 0xdb, 0x8d, 0x3c, 0xd1, 0x94, 0xcc, 0x6e, 0xe8, 0x0f, 0x7a, 0x81, 0x10, 0xe5, 0xa4, 0x28, 0x65, + 0xe0, 0x35, 0x28, 0xc8, 0x54, 0x8b, 0x53, 0xa6, 0xb6, 0xe2, 0x13, 0xff, 0x6d, 0x41, 0x69, 0x97, + 0x9e, 0x4b, 0x18, 0x31, 0x7a, 0x02, 0xf3, 0x6d, 0x4e, 0x83, 0x2e, 0x8d, 0xba, 0x52, 0xa9, 0xbc, + 0xf9, 0x46, 0x9a, 0xc2, 0x44, 0x6d, 0xc3, 0xe8, 0xb4, 0x02, 0x1e, 0x0d, 0x49, 0x62, 0x82, 0xb6, + 0x60, 0x4e, 0xf7, 0x84, 0xc4, 0x50, 0xde, 0xac, 0x4f, 0xb3, 0x4e, 0xda, 0x46, 0x18, 0x1b, 0x83, + 0x3b, 0x1f, 0x40, 0x65, 0xc4, 0xad, 0xc0, 0x7a, 0xc2, 0x86, 0xa6, 0x22, 0x27, 0x6c, 0x28, 0x72, + 0x77, 0x4a, 0xfd, 0x81, 0xca, 0x73, 0x9e, 0x28, 0x62, 0x2b, 0xf7, 0xbe, 0x75, 0x67, 0x0b, 0x16, + 0xb2, 0x5e, 0xaf, 0x63, 0x8b, 0xbf, 0x02, 0xb4, 0x13, 0x31, 0xca, 0x99, 0x84, 0xb7, 0xcb, 0xe2, + 0x98, 0x1e, 0xb3, 0xd9, 0x95, 0x56, 0xd5, 0xcb, 0x65, 0xab, 0xb7, 0x0e, 0x25, 0x37, 0x36, 0x07, + 0xb7, 0x65, 0x5f, 0xa6, 0x0c, 0xfc, 0x00, 0x50, 0x93, 0xf9, 0x8c, 0x33, 0x3d, 0x5f, 0x97, 0xf8, + 0xc7, 0x6d, 0x83, 0xe5, 0x6a, 0x5d, 0x74, 0x1f, 0xf2, 0x62, 0x3c, 0x25, 0x94, 0xf2, 0xe6, 0xad, + 0x34, 0xd3, 0xc9, 0x1c, 0x13, 0xa9, 0x80, 0x3d, 0xe3, 0x54, 0x8f, 0xf4, 0x15, 0x07, 0x9c, 0xd2, + 0xca, 0x26, 0x94, 0x3d, 0x1e, 0x2a, 0x59, 0x12, 0x3a, 0xd4, 0x53, 0x73, 0xd6, 0x9b, 0x86, 0xc2, + 0xc7, 0x09, 0x58, 0x31, 0xa9, 0x37, 0x01, 0xfb, 0x26, 0x14, 0xa4, 0xad, 0x46, 0x3b, 0xb1, 0x03, + 0x94, 0x14, 0x1f, 0x26, 0x50, 0x6f, 0x1a, 0x68, 0x25, 0x1b, 0xa8, 0x64, 0xfc, 0x7e, 0xa1, 0x75, + 0xc5, 0x4c, 0xef, 0x09, 0x1b, 0xe5, 0x49, 0x7e, 0xcf, 0xae, 0xd9, 0x58, 0x22, 0x85, 0x6f, 0xb1, + 0x04, 0x62, 0xc7, 0xae, 0xdb, 0xc2, 0xb7, 0x24, 0xf0, 0x23, 0x28, 0xb6, 0x3b, 0x2f, 0x59, 0x8f, + 0xa2, 0xb7, 0xc5, 0xa4, 0x75, 0xd9, 0x39, 0x8b, 0xf5, 0x9c, 0x2e, 0x8d, 0xd5, 0x9f, 0x18, 0x39, + 0xfe, 0xc1, 0xd2, 0x67, 0x9a, 0x81, 0xa8, 0x28, 0x63, 0xc7, 0x4e, 0x7e, 0x62, 0x65, 0x0a, 0x3e, + 0xd1, 0x62, 0xd4, 0x82, 0xaa, 0x1b, 0xf4, 0x07, 0xbc, 0xc9, 0xbe, 0xf6, 0x02, 0x8f, 0x7b, 0x61, + 0x10, 0x3b, 0x45, 0x69, 0xb2, 0x96, 0x0d, 0x3d, 0xa2, 0x41, 0x26, 0x4c, 0xf0, 0x77, 0x16, 0x2c, + 0x8d, 0x31, 0xaf, 0xc0, 0x95, 0xbb, 0x1c, 0xd7, 0x7b, 0xc9, 0xce, 0xb7, 0xa5, 0x62, 0x6d, 0x26, + 0x9a, 0xd1, 0x2b, 0xe0, 0x17, 0x0b, 0x56, 0xa6, 0x29, 0x4c, 0x45, 0x53, 0x03, 0x78, 0x11, 0x79, + 0x3d, 0x1a, 0x0d, 0x3f, 0x61, 0x43, 0x7d, 0xfd, 0x65, 0x38, 0xe8, 0x73, 0x58, 0x1d, 0xf3, 0xf5, + 0x51, 0x47, 0xa5, 0x48, 0x81, 0xba, 0x3b, 0x13, 0x94, 0xd2, 0x23, 0x33, 0xcc, 0xf1, 0x5f, 0x16, + 0xdc, 0x9e, 0x2a, 0x4a, 0x7b, 0xd2, 0xca, 0xf6, 0xe4, 0x03, 0xa8, 0x1e, 0x8a, 0xcd, 0xd6, 0x64, + 0x31, 0xf7, 0x02, 0x2a, 0x34, 0x75, 0xd3, 0x4e, 0xf0, 0x91, 0x0b, 0xf3, 0x92, 0xb7, 0x4b, 0xfb, + 0x1a, 0xe6, 0x3b, 0x57, 0xc0, 0xdc, 0x30, 0xfa, 0x7a, 0xf1, 0x1b, 0x52, 0x80, 0x91, 0x17, 0x91, + 0xb9, 0xd5, 0x24, 0x21, 0x56, 0xfa, 0x88, 0xc1, 0xb5, 0xd6, 0x72, 0x08, 0xeb, 0x66, 0x15, 0x8e, + 0x20, 0xb9, 0x7c, 0x52, 0x1f, 0x03, 0xa4, 0xaa, 0x7a, 0x03, 0x5c, 0xd2, 0x9f, 0x19, 0x65, 0xfc, + 0x1c, 0xd6, 0xcd, 0x9e, 0xbe, 0x46, 0x40, 0xd3, 0x2d, 0xb9, 0xb4, 0x5b, 0x70, 0x0b, 0xec, 0x03, + 0xe2, 0x8a, 0xbb, 0x5a, 0x4e, 0xab, 0x29, 0x91, 0xa6, 0x84, 0xc9, 0xf3, 0x30, 0xe6, 0xc6, 0x44, + 0x7c, 0x0b, 0xde, 0x8b, 0x30, 0xe2, 0x12, 0x71, 0x85, 0xc8, 0x6f, 0xfc, 0x25, 0xe4, 0xf7, 0xc2, + 0x2e, 0x43, 0x8b, 0x90, 0x73, 0x9b, 0xda, 0x47, 0xce, 0x6d, 0xa2, 0xbb, 0xd2, 0xbd, 0xde, 0x21, + 0x95, 0xf4, 0x70, 0x07, 0xc4, 0x25, 0x32, 0xf0, 0x3d, 0xa8, 0xb8, 0xf1, 0x4e, 0x18, 0x46, 0x5d, + 0x51, 0xea, 0x30, 0xd2, 0x77, 0xd2, 0x28, 0x13, 0x3f, 0x85, 0xaa, 0x70, 0xdf, 0xe6, 0x94, 0x27, + 0x9b, 0x7a, 0x15, 0x8a, 0x82, 0x97, 0x84, 0xd3, 0x94, 0xbc, 0xf7, 0x84, 0x9e, 0x59, 0x80, 0x92, + 0xc0, 0x9f, 0x2a, 0x0f, 0xad, 0x53, 0x16, 0xf0, 0x4c, 0x96, 0x24, 0x2d, 0x1d, 0x54, 0x88, 0x22, + 0x10, 0x56, 0x47, 0xd1, 0x98, 0x17, 0x53, 0xcc, 0x82, 0x4b, 0xa4, 0x0c, 0xff, 0x68, 0x01, 0x18, + 0x40, 0x83, 0x38, 0x31, 0xb1, 0x66, 0x9b, 0xa0, 0x77, 0x33, 0x6f, 0x97, 0xc9, 0x9d, 0x9a, 0x88, + 0x48, 0xe6, 0x85, 0xd3, 0x30, 0x2b, 0x54, 0x37, 0x47, 0x35, 0xd5, 0x57, 0x7c, 0x5d, 0x26, 0x71, + 0x6d, 0x56, 0x76, 0xfc, 0x41, 0xcc, 0x59, 0xa4, 0x11, 0x89, 0x37, 0x96, 0x62, 0x24, 0xf9, 0x49, + 0x19, 0xd3, 0x53, 0x84, 0xee, 0x41, 0x41, 0x20, 0x35, 0x7b, 0x60, 0xfc, 0x18, 0x4a, 0x88, 0xdb, + 0xfa, 0x26, 0x99, 0xba, 0x7b, 0x10, 0xe4, 0xe5, 0x8b, 0x5a, 0xb7, 0x8b, 0x7c, 0x4c, 0x57, 0xc1, + 0xde, 0xf5, 0x54, 0x7f, 0xdb, 0x44, 0x7c, 0x4a, 0x0e, 0x3d, 0x97, 0xf3, 0x27, 0x38, 0x54, 0xbc, + 0x25, 0x96, 0xd5, 0x00, 0x89, 0xbb, 0xe3, 0x26, 0xf7, 0x9b, 0x79, 0x94, 0xda, 0x99, 0x47, 0x69, + 0x1b, 0x96, 0xd5, 0x90, 0xfc, 0x97, 0x4e, 0x7f, 0xce, 0xc1, 0x32, 0x61, 0xb1, 0xf7, 0x8a, 0xb9, + 0x41, 0xcc, 0xa3, 0x41, 0xb2, 0xe0, 0x3e, 0x0e, 0x8f, 0x74, 0xaa, 0x6d, 0xa2, 0x88, 0xd7, 0xe9, + 0x24, 0xf4, 0x10, 0xca, 0xe3, 0xdd, 0x3f, 0xa9, 0x9a, 0x55, 0x41, 0x0f, 0x61, 0xae, 0x1d, 0x0e, + 0xa2, 0x4e, 0x72, 0x0d, 0xae, 0xa6, 0xda, 0x0a, 0x99, 0x12, 0x13, 0xa3, 0x96, 0xe9, 0xa3, 0xc2, + 0xe5, 0x7d, 0x84, 0x9e, 0x8c, 0xf5, 0x91, 0xfc, 0x73, 0x29, 0x6f, 0xfe, 0x3f, 0x35, 0x18, 0x11, + 0x93, 0x51, 0x6d, 0xfc, 0xbd, 0x05, 0x0b, 0x59, 0x08, 0xaf, 0x35, 0x18, 0x49, 0x45, 0x72, 0x53, + 0x2b, 0x62, 0x4f, 0xab, 0x48, 0x3e, 0xad, 0x48, 0xfa, 0xce, 0x2d, 0x64, 0xde, 0xb9, 0xf8, 0x04, + 0xd6, 0x26, 0xca, 0xb4, 0x13, 0xf6, 0xfa, 0xa2, 0x1f, 0xfe, 0x45, 0xb9, 0xc4, 0xca, 0x88, 0x22, + 0x5d, 0xa8, 0x12, 0x51, 0x04, 0x7e, 0x0c, 0xb7, 0xdb, 0x8c, 0x67, 0x8a, 0x64, 0xba, 0xad, 0x0e, + 0xf6, 0x1e, 0x3b, 0x9b, 0x71, 0x7c, 0x21, 0xc2, 0x1f, 0x82, 0x73, 0xd0, 0xef, 0x52, 0xce, 0x6e, + 0x64, 0xbd, 0x0d, 0xf3, 0xfb, 0x61, 0x3f, 0xf4, 0xc3, 0xe3, 0xe1, 0x15, 0x23, 0xef, 0xc0, 0x9c, + 0xda, 0x8f, 0xea, 0x91, 0x52, 0x22, 0x86, 0xc4, 0xb7, 0x44, 0x43, 0x77, 0xa8, 0xdf, 0x19, 0xf8, + 0x02, 0x86, 0xf8, 0xf7, 0x8a, 0xb7, 0xab, 0xbf, 0x5e, 0xd4, 0xac, 0xdf, 0x2e, 0x6a, 0xd6, 0x1f, + 0x17, 0x35, 0xeb, 0xa7, 0x3f, 0x6b, 0xff, 0x3b, 0x2a, 0xca, 0xbf, 0xf0, 0x47, 0xff, 0x04, 0x00, + 0x00, 0xff, 0xff, 0xc3, 0xb3, 0xdc, 0xe3, 0x96, 0x0f, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 7126811ac..bd3e52b26 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -3,7 +3,6 @@ syntax = "proto3"; package internal; message IndexMeta { - string TimeQuantum = 2; } message FrameMeta { diff --git a/server.go b/server.go index 049672686..f28ca3864 100644 --- a/server.go +++ b/server.go @@ -367,9 +367,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { idx.SetRemoteMaxSlice(obj.Slice) } case *internal.CreateIndexMessage: - opt := IndexOptions{ - TimeQuantum: TimeQuantum(obj.Meta.TimeQuantum), - } + opt := IndexOptions{} _, err := s.Holder.CreateIndex(obj.Index, opt) if err != nil { return err From 8c2387b45ee52667c85aeb22be26d0d65be423a7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 18 Apr 2018 11:06:53 -0500 Subject: [PATCH 38/79] remove references to Input Defintion from the docs --- docs/api-reference.md | 123 --------------------------------------- docs/getting-started.md | 9 --- docs/input-definition.md | 11 ---- 3 files changed, 143 deletions(-) delete mode 100644 docs/input-definition.md diff --git a/docs/api-reference.md b/docs/api-reference.md index 0c0314f85..f7c3641b9 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -198,129 +198,6 @@ curl localhost:10101/index/repository/frame/stats/field/pullrequests \ {} ``` -### Create input definition - -
-Input definition is deprecated as of v0.9. -
- -`POST /index//input-definition/` - -Creates an input definition in the given index with the given name. - -The request payload is JSON, and it must contain the fields `frames` and `fields`. `frames` is an array of frames used within this input definition. Each frame must contain a `name` and may contain the following options: - -* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`. -* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. -* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. - -The `fields` array contains a series of JSON objects describing how to process each field received in the input data. Each `field` object must contain a `name` which maps to the source JSON field name. One field must be defined at the `primaryKey`. The `primarykey` source field's value must be an unsigned integer which maps directly to a columnID in Pilosa. - -* `name` (string): Maps the source data field to actions that process the field's corresponding value. -* `actions` (array): List of actions that will process the field's value. - -The `action` describes how the field value will be processed. Each `action` may contain: - -* `frame` (string): The Frame that will contain this action's set bits. -* `rowid` (int): The action can use this as a pre-defined SetBit rowID. The user is required to ensure this ID does not overlap with other rows in use per frame. -* `valueDestination` (string): The mapping rule used for this data. - - `value-to-row`: The value should be an integer and will map directly to a RowID. - - `single-row-boolean`: If the value is true set a bit using the `rowid`. - - `mapping`: Map the value to a RowID in the `valueMap`. -* `valueMap` (object): string and integer pairs used to map field values to RowID's. - -``` request -curl localhost:10101/index/user/input-definition/stargazer-input \ - -X POST \ - -d '{ - "frames":[ - { - "name": "language", - "options": {"inverseEnabled": true} - } - ], - "fields":[ - { - "name": "repo_id", - "primaryKey":true - }, - { - "name": "language_id", - "actions":[ - { - "frame": "language", - "valueDestination": "mapping", - "valueMap": { - "Go": 5, - "Python": 17, - "C++": 10 - } - } - ] - } - ] - }' -``` -``` response -{} -``` - -### Get input definition - -
-Input definition is deprecated as of v0.9. -
- -`GET /index//input-definition/` - -Returns the given input definition as JSON. - -``` request -curl -XGET localhost:10101/index/user/input-definition/stargazer-input -``` -``` response -{"frames":[{"name":"language","options":{"inverseEnabled":true}}],"fields":[{"name":"repo_id","primaryKey":true},{"name":"language_id","actions":[{"frame":"language","valueDestination":"mapping","valueMap":{"Go":5,"Python":17,"C++":10}}]}]} -``` - -### Remove input definition - -
-Input definition is deprecated as of v0.9. -
- -`DELETE /index//input-definition/` - -Removes the given input definition. - -``` request -curl -XDELETE localhost:10101/index/user/input-definition/stargazer-input -``` -``` response -{} -``` - -### Process input data - -
-Input definition is deprecated as of v0.9. -
- -`POST /index//input/` - -Processes the JSON payload using the given input definition. - -The request payload is a JSON array of objects containing one field for the primary key that corresponds to the column, and additional fields that will be handled by corresponding actions in the input definition. - -``` request -curl localhost:10101/index/user/input/stargazer-input \ - -X POST \ - -d '[{"language_id": "Go", "repo_id": 92274475}]' -``` -``` response -{} -``` - ### List hosts `GET /hosts` diff --git a/docs/getting-started.md b/docs/getting-started.md index c9cdb8aba..bee165643 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -4,7 +4,6 @@ weight = 3 nav = [ "Starting Pilosa", "Sample Project", - "Input Definition", "What's Next?", ] +++ @@ -111,14 +110,6 @@ docker exec -it pilosa /pilosa import -i repository -f language /language.csv Note that both the user IDs and the repository IDs were remapped to sequential integers in the data files, they don't correspond to actual Github IDs anymore. You can check out [languages.txt](https://github.com/pilosa/getting-started/blob/master/languages.txt) to see the mapping for languages. -### Input Definition - -
-Input definition is deprecated as of v0.9. -
- -Alternatively Pilosa can import JSON data using an [Input Definition](../input-definition/) describing the schema and ETL rules to process the data. - #### Make Some Queries
diff --git a/docs/input-definition.md b/docs/input-definition.md deleted file mode 100644 index 9d0e5fe6b..000000000 --- a/docs/input-definition.md +++ /dev/null @@ -1,11 +0,0 @@ -+++ -title = "Input Definition" -weight = 8 -+++ - -## Input Definition -
-Input definition is deprecated as of Pilosa v0.9.
-
-The previous version of this page is still available here. -
From ff8fea804c17abf7f081fec94b9c4ad88a0b0bf6 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Wed, 18 Apr 2018 11:22:59 -0500 Subject: [PATCH 39/79] remove outdated pdk docs, and write some new ones incomplete, but an improvement? --- docs/pdk.md | 111 +++++++++++++++++++++++++++------------------------- 1 file changed, 58 insertions(+), 53 deletions(-) diff --git a/docs/pdk.md b/docs/pdk.md index 8c02dc279..a9df97832 100644 --- a/docs/pdk.md +++ b/docs/pdk.md @@ -2,66 +2,71 @@ title = "PDK" weight = 11 nav = [ + "Examples and Executables", "Library", - "Examples", ] +++ ## PDK -The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains Go libraries to help you use Pilosa effectively. From importing data quickly, to managing the mappings from contiguous integer ids to values of other types, the PDK should help you get off the ground quickly. +The [Pilosa Dev Kit](https://github.com/pilosa/pdk) contains executables, examples, and Go libraries to help you use Pilosa effectively. + +### Examples and Executables +Running `pdk -h` will give the most up to date list of all the tools and examples that PDK provides. We'll cover a few of the more important ones here. + +#### Kafka +`pdk kafka` reads either JSON or Avro encoded records from Kafka (using the +Confluent Schema Registry in the case of Avro), and indexes them in Pilosa. Each +record from Kafka is assigned a Pilosa column, and each value in a record is +assigned a row or field. Frame and field names are built from the "path" through +the record to arrive at that field. For example: + +```json +{ + "name": "jill", + "favorite_foods": ["corn chips", "chipotle dip"], + "location": { + "city": "Austin", + "state": "Texas", + "latitude": 3754, + "longitude": 4526 + }, + "active": true, + "age": 27 +} +``` + +This JSON object would result in the following Pilosa schema: + +| Name | Field | Type | Size/Min | Max | +|----------------|-----------|--------|----------|------------| +| name | | ranked | 100000 | | +| favorite_foods | | ranked | 100000 | | +| default | | Ranked | 100000 | | +| | age | int | 0 | 2147483647 | +| location | | ranked | 1000 | | +| | latitude | int | 0 | 2147483647 | +| | longitude | int | 0 | 2147483647 | +| location-city | | ranked | 100000 | | +| location-state | | ranked | 100000 | | + +All frames are created as ranked frames by default, and fields are created with +a minmum size of zero and a fixed maximum of 2147483647. Fields at the top level +are created in the default frame. Frames are a dash-separated concatenation of +all key values in the path - you can see this with frames like location-city. + + +Most the options to `pdk kafka` are self-explanatory (kafka hosts, pilosa hosts, +kafka topics, kafka group, etc.), but there are a few options that give some +control over the way data is indexed, and ingestion performance. + +* `--batch-size`: The batch size control how many set bits or values are batched up to be imported *per frame*. So for fields that have one value per record, you have to wait for `batch-size` records to come through before you'll see the data indexed in Pilosa. Fields like `favorite_foods` which can have multiple values could be indexed sooner. +* `--framer.collapse`: This is a list of strings which will be removed from the frame names created by dash-concatentating all names in the JSON path to a value. E.G. if "location" were listed in `framer.collapse`, then there would be frames named "city" and "state" rather than "location-city" and "location-state". +* `--framer.ignore`: This allows you to skip indexing on any path containing these strings. If you have a field like email address or some other unique ID, you might not want to index it. +* `--subject-path`: If nothing is passed for this option, then each record will be assigned a unique sequential column ID. If `subject-path` is specified, then the value at this path in the record will be mapped to a column ID. If the same value appears in another record, the same column ID will be used. +* `--proxy`: The PDK ingests data, but also keeps a mapping for string values to row IDs, and from subjects to column ids. Because of this, querying Pilosa directly may not be useful, since it only returns integer row and column ids. The PDK will start a proxy server which intercepts requests to Pilosa using strings for row and column ids, and translates them to the integers that Pilosa understands. It will also translate responses so that (e.g.) a TopN query will return `{"results":[[{"Key":"chipotle dip","Count":1},{"Key":"corn chips","Count":1}]]}`. By default, the mapping is stored in an embedded leveldb. -The PDK also contains some fully worked examples which make use of its tools. These are available in the `usecase` subdirectory and can be run as subcommands of the `pdk` binary. ### Library -#### Mapping - -Importing data into Pilosa is dependent on mapping it to integer IDs. PDK provides some predefined functions for inline mapping to simplify this process, supported by a framework for linking these mappings with the associated fields in a source CSV file. If no custom mapping is necessary, the entire import process can be described by an import definition file. The file is composed of four main parts: - -* an enumeration of field names -* a list of parsers that are used to parse strings in the CSV to values -* a list of commonly used, named, mapper functions -* a list of ParserMappers - objects that encapsulate all of the work related to a single frame. - -This definition file can quickly get long, and defining it manually would be quite tedious. That's why we have a tool to generate a definition file by looking at a data set. This will handle most of the legwork, but since it can only guess at the application, it uses the simplest mappings - each column gets mapped to one frame in an appropriate way. This is intended as a starting point, to be updated to suit your use of the PDK. - -With this definition available, the PDK tool can run the import, which consists of these steps: - -- create the index -- create all frames -- for each CSV file, read all rows -- for each CSV record: - - generate a columnID - - apply all ParserMappers, generating a list of (frame, ID) pairs - - set the appropriate bit. schematically: SetBit(id=rowID, frame=frame, profileID=columnID) - -The process is summarized in this flowchart: - -![Bitmapper flowchart](/img/docs/pdk-bitmapper-flowchart.svg) - - -Some of the simple mapper functions available with PDK include: - -* YearMapper: Maps a `time.Time` value to an integer equal to the `Time`'s year. -* MonthMapper: Maps a `time.Time` value to an integer equal to the `Time`'s month, in [0, 11]. -* DayOfWeekMapper: Maps a `time.Time` value to an integer equal to the `Time`'s day of the week, in [0, 6]. -* HourMapper: Maps a `time.Time` value to an integer equal to the `Time`'s hour, in [0, 23]. -* TimeOfDayMapper: Maps a `time.Time` value to the range [0, N-1], where N is specified by `Res`. This is useful if the resolution used by HourMapper is too small (or large). For example, TimeOfDayMapper with `Res`=48 maps to 48 half-hour bins. -* BoolMapper: Maps a boolean value to the range [0, 1]. -* IntMapper: Maps an integer value to the range [Min, Max]. This is suitable for a field with a small- to moderate-sized domain. -* SparseIntMapper: Maps integer values through an arbitrary table, foreign keys for example. This is suitable if the table size is small. -* LinearFloatMapper: Maps floating point values through a linear function. Inputs in the range [`Min`, `Max`] are mapped to row IDs in the range [0, `Res - 1`], where each ID represents one of `Res` evenly spaced buckets. -* FloatMapper: Maps floating point values using arbitrary buckets, in case even spacing is not suitable. These buckets are specified with an array of floats representing the left end of each bucket. -* GridMapper: Maps a pair of floats to a single integer, identifying a cell in a rectangular grid. This can be used, for example, to represent (latitude, longitude) location coarsely, as in the taxi data example. -* CustomMapper: When none of the predefined mappers will work, or when multiple fields determine a row ID value, an arbitrary mapping function can be used. Define a function in Go, with the necessary behavior, and wrap it in a CustomMapper. - -### Examples - -Run `make install` to build and install the `pdk` binary which contains all the examples. Just running `pdk` will bring up a list of all the examples, with a brief description of each. `pdk help ` will bring up a more detailed description of that example along with all arguments that it accepts to configure its functionality. - - +For now, the [Godocs](https://godoc.org/github.com/pilosa/pdk) have the most up to date library documentation. From 72eb3d239ac264a69ab32c7a60e100a675327f18 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 18 Apr 2018 14:18:57 -0500 Subject: [PATCH 40/79] Remove references to inverse frames --- docs/administration.md | 1 - docs/api-reference.md | 5 +---- docs/data-model.md | 16 +--------------- docs/examples.md | 4 ++-- docs/glossary.md | 2 +- docs/query-language.md | 13 +------------ docs/webui.md | 4 ++-- 7 files changed, 8 insertions(+), 37 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 9d8c5b70c..c9d932df0 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -201,7 +201,6 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi - **NumCPU:** Number of Cores per Node - **BSIEnabled:** Bit Slice Index Frames in use. - **TimeQuantumEnabled:** Time Quantum Frames in use. -- **InverseEnabled:** Inverse Frames in use. - **NumIndexes:** Number of Indexes in the Cluster. - **NumFrames:** Number of Frames in the Cluster. - **NumSlices:** Number of Slices in the Cluster. diff --git a/docs/api-reference.md b/docs/api-reference.md index f7c3641b9..a2697f311 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -103,7 +103,6 @@ Creates a frame in the given index with the given name. The request payload is in JSON, and may contain the `options` field. The `options` field is a JSON object which may contain the following fields: * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. -* `inverseEnabled` (boolean): Enables [the inverted view](../data-model/#inverse) for this frame if `true`. * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. * `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields. @@ -119,9 +118,7 @@ Each individual `field` contains the following: Integer fields are stored as n-bit range-encoded values. Pilosa supports 63-bit, signed integers with values between `min` and `max`. ``` request -curl localhost:10101/index/user/frame/language \ - -X POST \ - -d '{"options": {"inverseEnabled": true}}' +curl localhost:10101/index/user/frame/language -X POST ``` ``` response {} diff --git a/docs/data-model.md b/docs/data-model.md index 6a57a96a1..dc8646a85 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -77,26 +77,12 @@ Columns are sharded on a preset width, and each shard is referred to as a Slice. ### View -Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. The Inverse View contains the same data with the axes inverted.Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation. +Views represent the various data layouts within a Frame. The primary View is called Standard, and it contains the typical Row and Column data. Time-based Views are automatically generated for each time quantum. Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface from the physical data representation. #### Standard The standard View contains the same Row/Column format as the input data. -#### Inverse - -The Inverse View contains the same data with the Row and Column swapped. - -For example, the following `SetBit()` queries will result in the data described in the illustration below: -``` -SetBit(frame="A", rowID=8, columnID=3) -SetBit(frame="A", rowID=11, columnID=3) -SetBit(frame="A", rowID=19, columnID=5) -``` - -![inverse frame diagram](/img/docs/frame-inverse.svg) -*Inverse frame diagram* - #### Time Quantums If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below: diff --git a/docs/examples.md b/docs/examples.md index 225e0f07c..f2d458b30 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -205,7 +205,7 @@ For more examples and details, see this [ipython notebook](https://github.com/pi The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. -We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. There are examples where --- tried to solve this chemical similarity search problem using other indexes (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. +We'd like to use Pilosa to search through millions of molecules and find those most similar to a given molecule. Others have tried to solve this chemical similarity search problem using databases (MongoDB, PostgreSQL), so it will be interesting to compare those results to Pilosa using the same data set. Calculation of the similarity of any two molecules is achieved by comparing their molecular fingerprints. These fingerprints are comprised of structural information about the molecule which has been encoded as a series of bits. The most commonly used algorithm to calculate the similarity is the Tanimoto coefficient. ``` @@ -218,7 +218,7 @@ All source code to calculate tanimoto for molecule fingerprint using Pilosa is a #### Data model -We use the latest ChEMBL release chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. +We use the [latest ChEMBL release](ftp://ftp.ebi.ac.uk/pub/databases/chembl/ChEMBLdb/releases/) chembl_22.sdf for test data. Each molecule in the SD file gives us the canonical isomeric SMILES (Simplified molecular-input line-entry system) and chembl_id. Because Pilosa store information as a series of bits, we use RDKit in Python to convert molecules from their SMILES encoding to Morgan fingerprints, which are arrays of “on” bit positions. diff --git a/docs/glossary.md b/docs/glossary.md index 612c65dd4..d222cb511 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -64,4 +64,4 @@ nav = [] [TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). -[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The two primary views are standard and inverse which represent the typical [row](#row)/[column](#column) data and its inverse respectively (an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), or a matrix transpose). Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. +[View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. diff --git a/docs/query-language.md b/docs/query-language.md index 6b0584320..f26cf021d 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -386,13 +386,12 @@ Return `2` ``` TopN([BITMAP_CALL], , [n=UINT], - [inverse=true], [, ]) + [, ]) ``` **Description:** Return the id and count of the top `n` bitmaps (by count of bits) in the frame. -`inverse=true` specifies that the call should operate on the [inverse view ](../data-model/#inverse). The `field` and `filters` arguments work together to only return Bitmaps which have the attribute specified by `field` with one of the values specified in `filters`. @@ -419,16 +418,6 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1} * count is amount of repositories * Results are the number of repositories that each user starred in descending order for all users in the stargazer frame, for example user 1 starred two repositories, user 2 starred two repositories, user 3 starred one repository. -``` -TopN(frame="stargazer", inverse=true) -``` - -Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}, {"key": 3, "count": 1}]` - -* key is a repository ID -* count is amount of users -* Results are the number of users that starred each repository in descending order for all respositories in the stargazer frame. - ``` TopN(frame="stargazer", n=2) ``` diff --git a/docs/webui.md b/docs/webui.md index 738fb20fd..0f14934b4 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -32,9 +32,9 @@ In addition to standard PQL, the console supports a few special commands, prefix - `:create frame ` - `:delete frame ` -Frame creation also supports options like `timeQuantum` or `inverseEnabled`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame). +Frame creation also supports options like `timeQuantum`. When creating a new frame, add options by using the keys documented in [API reference](../api-reference/#create-frame). -- `:create frame inverseEnabled=true cacheSize=10000` +- `:create frame cacheSize=10000` ### Cluster Admin From 84e6493d7a94a6db7a6ffcea79455a9d44b08d3f Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 19 Apr 2018 10:31:01 -0500 Subject: [PATCH 41/79] Add deprecation warning to examples doc --- docs/examples.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/examples.md b/docs/examples.md index f2d458b30..d7830a2a8 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -201,6 +201,10 @@ For more examples and details, see this [ipython notebook](https://github.com/pi ### Chemical similarity search +
+This example uses the inverse frames feature, which is deprecated as of v0.9.0. This will soon be updated to reflect the current Pilosa API. +
+ #### Overview The notion of chemical similarity (or molecular similarity) plays an important role in predicting the properties of chemical compounds, designing chemicals with a predefined set of properties, and—especially—conducting drug design studies. All of these are accomplished by screening large indexes containing structures of available or potentially available chemicals. From 876ed56e306f57c08b49688a3186d412bb2061c1 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 19 Apr 2018 14:51:42 -0500 Subject: [PATCH 42/79] move pilosa.Config to pilosa/server.Config step 1 of #1203 The Config object is really just a specification of the options to pilosa server, so it makes sense to have it in that package. --- broadcast_test.go | 5 +- cluster.go | 5 +- cmd/server_test.go | 8 +- config.go | 226 ------------------------ ctl/backup.go | 5 +- ctl/bench.go | 5 +- ctl/common.go | 3 +- ctl/config.go | 3 +- ctl/config_test.go | 4 +- ctl/export.go | 5 +- ctl/import.go | 5 +- ctl/restore.go | 5 +- gossip/gossip.go | 105 ++++++++--- pilosa.go | 4 - server.go | 25 ++- server/cluster_test.go | 4 +- server/config.go | 132 ++++++++++++++ config_test.go => server/config_test.go | 17 +- server/server.go | 14 +- server/server_test.go | 5 +- toml/toml.go | 30 ++++ 21 files changed, 308 insertions(+), 307 deletions(-) delete mode 100644 config.go create mode 100644 server/config.go rename config_test.go => server/config_test.go (78%) create mode 100644 toml/toml.go diff --git a/broadcast_test.go b/broadcast_test.go index 8f0245c88..f4ff4c0fa 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -53,7 +53,10 @@ func testMessageMarshal(t *testing.T, m proto.Message) { // Ensure that BroadcastReceiver can register a BroadcastHandler. func TestBroadcast_BroadcastReceiver(t *testing.T) { - s := pilosa.NewServer() + s, err := pilosa.NewServer() + if err != nil { + t.Fatalf("getting new server: %v", err) + } sbr := NewSimpleBroadcastReceiver() sbh := NewSimpleBroadcastHandler() diff --git a/cluster.go b/cluster.go index 7e871698d..be668a6bc 100644 --- a/cluster.go +++ b/cluster.go @@ -40,9 +40,6 @@ const ( // DefaultPartitionN is the default number of partitions in a cluster. DefaultPartitionN = 256 - // DefaultReplicaN is the default number of replicas per partition. - DefaultReplicaN = 1 - // ClusterState represents the state returned in the /status endpoint. ClusterStateStarting = "STARTING" ClusterStateNormal = "NORMAL" @@ -276,7 +273,7 @@ func NewCluster() *Cluster { return &Cluster{ Hasher: &jmphasher{}, PartitionN: DefaultPartitionN, - ReplicaN: DefaultReplicaN, + ReplicaN: 1, EventReceiver: NopEventReceiver, joiningLeavingNodes: make(chan nodeAction, 10), // buffered channel diff --git a/cmd/server_test.go b/cmd/server_test.go index b1f39e921..e69ce1134 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -21,9 +21,9 @@ import ( "testing" "time" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/cmd" _ "github.com/pilosa/pilosa/test" + "github.com/pilosa/pilosa/toml" ) func TestServerHelp(t *testing.T) { @@ -65,7 +65,7 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Bind, "localhost:10111") v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) - v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90)) + v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) return v.Error() }, @@ -86,7 +86,7 @@ func TestServerConfig(t *testing.T) { validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) - v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*9)) + v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9)) return v.Error() }, }, @@ -113,7 +113,7 @@ func TestServerConfig(t *testing.T) { validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"}) - v.Check(cmd.Server.Config.AntiEntropy.Interval, pilosa.Duration(time.Minute*11)) + v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11)) v.Check(cmd.Server.CPUProfile, profFile.Name()) v.Check(cmd.Server.CPUTime, time.Minute) v.Check(cmd.Server.Config.LogPath, logFile.Name()) diff --git a/config.go b/config.go deleted file mode 100644 index 483965916..000000000 --- a/config.go +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "time" -) - -// Cluster types. -const ( - ClusterNone = "" - ClusterStatic = "static" - ClusterGossip = "gossip" -) - -// TLSConfig contains TLS configuration -type TLSConfig struct { - // CertificatePath contains the path to the certificate (.crt or .pem file) - CertificatePath string `toml:"certificate-path"` - // CertificateKeyPath contains the path to the certificate key (.key file) - CertificateKeyPath string `toml:"certificate-key-path"` - // SkipVerify disables verification for self-signed certificates - SkipVerify bool `toml:"skip-verify"` -} - -// Config represents the configuration for the command. -type Config struct { - // DataDir is the directory where Pilosa stores both indexed data and - // running state such as cluster topology information. - DataDir string `toml:"data-dir"` - // Bind is the host:port on which Pilosa will listen. - Bind string `toml:"bind"` - - // MaxWritesPerRequest limits the number of mutating commands that can be in - // a single request to the server. This includes SetBit, ClearBit, - // SetRowAttrs & SetColumnAttrs. - MaxWritesPerRequest int `toml:"max-writes-per-request"` - - // LogPath configures where Pilosa will write logs. - LogPath string `toml:"log-path"` - - // Verbose toggles verbose logging which can be useful for debugging. - Verbose bool `toml:"verbose"` - - // TLS - TLS TLSConfig - - Cluster struct { - // Disabled controls whether clustering functionality is enabled. - Disabled bool `toml:"disabled"` - Coordinator bool `toml:"coordinator"` - ReplicaN int `toml:"replicas"` - Hosts []string `toml:"hosts"` - LongQueryTime Duration `toml:"long-query-time"` - } `toml:"cluster"` - - // Gossip config is based around memberlist.Config. - Gossip struct { - // Port indicates the port to which pilosa should bind for internal state sharing. - Port string `toml:"port"` - Seeds []string `toml:"seeds"` - Key string `toml:"key"` - // StreamTimeout is the timeout for establishing a stream connection with - // a remote node for a full state sync, and for stream read and write - // operations. Maps to memberlist TCPTimeout. - StreamTimeout Duration `toml:"stream-timeout"` - // SuspicionMult is the multiplier for determining the time an - // inaccessible node is considered suspect before declaring it dead. - // The actual timeout is calculated using the formula: - // - // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval - // - // This allows the timeout to scale properly with expected propagation - // delay with a larger cluster size. The higher the multiplier, the longer - // an inaccessible node is considered part of the cluster before declaring - // it dead, giving that suspect node more time to refute if it is indeed - // still alive. - SuspicionMult int `toml:"suspicion-mult"` - // PushPullInterval is the interval between complete state syncs. - // Complete state syncs are done with a single node over TCP and are - // quite expensive relative to standard gossiped messages. Setting this - // to zero will disable state push/pull syncs completely. - // - // Setting this interval lower (more frequent) will increase convergence - // speeds across larger clusters at the expense of increased bandwidth - // usage. - PushPullInterval Duration `toml:"push-pull-interval"` - // ProbeInterval and ProbeTimeout are used to configure probing behavior - // for memberlist. - // - // ProbeInterval is the interval between random node probes. Setting - // this lower (more frequent) will cause the memberlist cluster to detect - // failed nodes more quickly at the expense of increased bandwidth usage. - // - // ProbeTimeout is the timeout to wait for an ack from a probed node - // before assuming it is unhealthy. This should be set to 99-percentile - // of RTT (round-trip time) on your network. - ProbeInterval Duration `toml:"probe-interval"` - ProbeTimeout Duration `toml:"probe-timeout"` - - // Interval and Nodes are used to configure the gossip - // behavior of memberlist. - // - // Interval is the interval between sending messages that need - // to be gossiped that haven't been able to piggyback on probing messages. - // If this is set to zero, non-piggyback gossip is disabled. By lowering - // this value (more frequent) gossip messages are propagated across - // the cluster more quickly at the expense of increased bandwidth. - // - // Nodes is the number of random nodes to send gossip messages to - // per Interval. Increasing this number causes the gossip messages - // to propagate across the cluster more quickly at the expense of - // increased bandwidth. - // - // ToTheDeadTime is the interval after which a node has died that - // we will still try to gossip to it. This gives it a chance to refute. - Interval Duration `toml:"interval"` - Nodes int `toml:"nodes"` - ToTheDeadTime Duration `toml:"to-the-dead-time"` - } `toml:"gossip"` - - AntiEntropy struct { - Interval Duration `toml:"interval"` - } `toml:"anti-entropy"` - - Metric struct { - // Service can be statsd, expvar, or none. - Service string `toml:"service"` - // Host tells the statsd client where to write. - Host string `toml:"host"` - PollInterval Duration `toml:"poll-interval"` - // Diagnostics toggles sending some limited diagnostic information to - // Pilosa's developers. - Diagnostics bool `toml:"diagnostics"` - } `toml:"metric"` -} - -// NewConfig returns an instance of Config with default options. -func NewConfig() *Config { - c := &Config{ - DataDir: "~/.pilosa", - Bind: ":10101", - MaxWritesPerRequest: 5000, - // LogPath: "", - // Verbose: false, - TLS: TLSConfig{}, - } - - // Cluster config. - c.Cluster.Disabled = false - // c.Cluster.Coordinator = false - c.Cluster.ReplicaN = DefaultReplicaN - c.Cluster.Hosts = []string{} - c.Cluster.LongQueryTime = Duration(time.Minute) - - // Gossip config. - c.Gossip.Port = "14000" - // c.Gossip.Seeds = []string{} - // c.Gossip.Key = "" - c.Gossip.StreamTimeout = Duration(10 * time.Second) - c.Gossip.SuspicionMult = 4 - c.Gossip.PushPullInterval = Duration(30 * time.Second) - c.Gossip.ProbeInterval = Duration(1 * time.Second) - c.Gossip.ProbeTimeout = Duration(500 * time.Millisecond) - c.Gossip.Interval = Duration(200 * time.Millisecond) - c.Gossip.Nodes = 3 - c.Gossip.ToTheDeadTime = Duration(30 * time.Second) - - // AntiEntropy config. - c.AntiEntropy.Interval = Duration(10 * time.Minute) - - // Metric config. - c.Metric.Service = "none" - // c.Metric.Host = "" - c.Metric.PollInterval = Duration(0 * time.Minute) - c.Metric.Diagnostics = true - - return c -} - -// Validate that all configuration permutations are compatible with each other. -func (c *Config) Validate() error { - if !c.Cluster.Disabled && len(c.Cluster.Hosts) > 0 { - return ErrConfigClusterEnabledHosts - } - return nil -} - -// Duration is a TOML wrapper type for time.Duration. -type Duration time.Duration - -// String returns the string representation of the duration. -func (d Duration) String() string { return time.Duration(d).String() } - -// UnmarshalText parses a TOML value into a duration value. -func (d *Duration) UnmarshalText(text []byte) error { - v, err := time.ParseDuration(string(text)) - if err != nil { - return err - } - - *d = Duration(v) - return nil -} - -// MarshalText writes duration value in text format. -func (d Duration) MarshalText() (text []byte, err error) { - return []byte(d.String()), nil -} - -// MarshalTOML write duration into valid TOML. -func (d Duration) MarshalTOML() ([]byte, error) { - return []byte(d.String()), nil -} diff --git a/ctl/backup.go b/ctl/backup.go index 8e725b41e..1b16391e0 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -21,6 +21,7 @@ import ( "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // BackupCommand represents a command for backing up a view. @@ -39,7 +40,7 @@ type BackupCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewBackupCommand returns a new instance of BackupCommand. @@ -88,6 +89,6 @@ func (cmd *BackupCommand) TLSHost() string { return cmd.Host } -func (cmd *BackupCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/bench.go b/ctl/bench.go index 01e07cc14..9e37fb704 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -24,6 +24,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/server" ) // BenchCommand represents a command for benchmarking index operations. @@ -42,7 +43,7 @@ type BenchCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewBenchCommand returns a new instance of BenchCommand. @@ -110,6 +111,6 @@ func (cmd *BenchCommand) TLSHost() string { return cmd.Host } -func (cmd *BenchCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *BenchCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/common.go b/ctl/common.go index 9bba68657..6ebcd3132 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -18,13 +18,14 @@ import ( "crypto/tls" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" "github.com/spf13/pflag" ) // CommandWithTLSSupport is the interface for commands which has TLS settings type CommandWithTLSSupport interface { TLSHost() string - TLSConfiguration() pilosa.TLSConfig + TLSConfiguration() server.TLSConfig } // SetTLSConfig creates common TLS flags diff --git a/ctl/config.go b/ctl/config.go index 2047941c7..7277f74ea 100644 --- a/ctl/config.go +++ b/ctl/config.go @@ -21,12 +21,13 @@ import ( toml "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // ConfigCommand represents a command for printing a default config. type ConfigCommand struct { *pilosa.CmdIO - Config *pilosa.Config + Config *server.Config } // NewConfigCommand returns a new instance of ConfigCommand. diff --git a/ctl/config_test.go b/ctl/config_test.go index b54446e34..ca08c273b 100644 --- a/ctl/config_test.go +++ b/ctl/config_test.go @@ -22,7 +22,7 @@ import ( "strings" "testing" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) func TestConfigCommand_Run(t *testing.T) { @@ -30,7 +30,7 @@ func TestConfigCommand_Run(t *testing.T) { stdin := bytes.NewReader(rder) r, w, _ := os.Pipe() cm := NewConfigCommand(stdin, w, os.Stderr) - cm.Config = pilosa.NewConfig() + cm.Config = server.NewConfig() err := cm.Run(context.Background()) w.Close() diff --git a/ctl/export.go b/ctl/export.go index 84b91edec..8c219a078 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -21,6 +21,7 @@ import ( "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // ExportCommand represents a command for bulk exporting data from a server. @@ -38,7 +39,7 @@ type ExportCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewExportCommand returns a new instance of ExportCommand. @@ -114,6 +115,6 @@ func (cmd *ExportCommand) TLSHost() string { return cmd.Host } -func (cmd *ExportCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *ExportCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/import.go b/ctl/import.go index 8103bf673..c3c117eb3 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -27,6 +27,7 @@ import ( "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // ImportCommand represents a command for bulk importing data. @@ -66,7 +67,7 @@ type ImportCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewImportCommand returns a new instance of ImportCommand. @@ -447,6 +448,6 @@ func (cmd *ImportCommand) TLSHost() string { return cmd.Host } -func (cmd *ImportCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *ImportCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/ctl/restore.go b/ctl/restore.go index 38528c2b5..89ee8ac8d 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -21,6 +21,7 @@ import ( "os" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" ) // RestoreCommand represents a command for restoring a frame from a backup. @@ -39,7 +40,7 @@ type RestoreCommand struct { // Standard input/output *pilosa.CmdIO - TLS pilosa.TLSConfig + TLS server.TLSConfig } // NewRestoreCommand returns a new instance of RestoreCommand. @@ -81,6 +82,6 @@ func (cmd *RestoreCommand) TLSHost() string { return cmd.Host } -func (cmd *RestoreCommand) TLSConfiguration() pilosa.TLSConfig { +func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig { return cmd.TLS } diff --git a/gossip/gossip.go b/gossip/gossip.go index a27da24c8..8ad00a8a7 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -29,6 +29,7 @@ import ( "github.com/hashicorp/memberlist" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) @@ -167,7 +168,7 @@ func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { } // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { +func NewGossipMemberSet(name string, host string, cfg Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { g := &GossipMemberSet{ Logger: server.Logger, @@ -181,17 +182,11 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, } if g.transport == nil { - port, err := strconv.Atoi(cfg.Gossip.Port) + port, err := strconv.Atoi(cfg.Port) if err != nil { return nil, fmt.Errorf("convert port: %s", err) } - bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) - if err != nil { - return nil, fmt.Errorf("getting uri from bind address: %s", err) - } - host := bindURI.Host() - // Set up the transport. transport, err := NewTransport(host, port, g.logger) if err != nil { @@ -203,15 +198,10 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, port := g.transport.Net.GetAutoBindPort() - bindURI, err := pilosa.NewURIFromAddress(cfg.Bind) - if err != nil { - return nil, fmt.Errorf("getting uri from bind address (with transport): %s", err) - } - host := bindURI.Host() - var gossipKey []byte - if cfg.Gossip.Key != "" { - gossipKey, err = ioutil.ReadFile(cfg.Gossip.Key) + var err error + if cfg.Key != "" { + gossipKey, err = ioutil.ReadFile(cfg.Key) if err != nil { return nil, fmt.Errorf("reading gossip key: %s", err) } @@ -226,14 +216,14 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, conf.AdvertisePort = port conf.AdvertiseAddr = pilosa.HostToIP(host) // - conf.TCPTimeout = time.Duration(cfg.Gossip.StreamTimeout) - conf.SuspicionMult = cfg.Gossip.SuspicionMult - conf.PushPullInterval = time.Duration(cfg.Gossip.PushPullInterval) - conf.ProbeTimeout = time.Duration(cfg.Gossip.ProbeTimeout) - conf.ProbeInterval = time.Duration(cfg.Gossip.ProbeInterval) - conf.GossipNodes = cfg.Gossip.Nodes - conf.GossipInterval = time.Duration(cfg.Gossip.Interval) - conf.GossipToTheDeadTime = time.Duration(cfg.Gossip.ToTheDeadTime) + conf.TCPTimeout = time.Duration(cfg.StreamTimeout) + conf.SuspicionMult = cfg.SuspicionMult + conf.PushPullInterval = time.Duration(cfg.PushPullInterval) + conf.ProbeTimeout = time.Duration(cfg.ProbeTimeout) + conf.ProbeInterval = time.Duration(cfg.ProbeInterval) + conf.GossipNodes = cfg.Nodes + conf.GossipInterval = time.Duration(cfg.Interval) + conf.GossipToTheDeadTime = time.Duration(cfg.ToTheDeadTime) // conf.Delegate = g conf.SecretKey = gossipKey @@ -242,7 +232,7 @@ func NewGossipMemberSet(name string, cfg *pilosa.Config, server *pilosa.Server, g.config = &gossipConfig{ memberlistConfig: conf, - gossipSeeds: cfg.Gossip.Seeds, + gossipSeeds: cfg.Seeds, } g.statusHandler = server @@ -526,3 +516,68 @@ func newTransport(conf *memberlist.Config) (*memberlist.NetTransport, error) { return nt, nil } + +// Config holds toml-friendly memberlist configuration. +type Config struct { + // Port indicates the port to which pilosa should bind for internal state sharing. + Port string `toml:"port"` + Seeds []string `toml:"seeds"` + Key string `toml:"key"` + // StreamTimeout is the timeout for establishing a stream connection with + // a remote node for a full state sync, and for stream read and write + // operations. Maps to memberlist TCPTimeout. + StreamTimeout toml.Duration `toml:"stream-timeout"` + // SuspicionMult is the multiplier for determining the time an + // inaccessible node is considered suspect before declaring it dead. + // The actual timeout is calculated using the formula: + // + // SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval + // + // This allows the timeout to scale properly with expected propagation + // delay with a larger cluster size. The higher the multiplier, the longer + // an inaccessible node is considered part of the cluster before declaring + // it dead, giving that suspect node more time to refute if it is indeed + // still alive. + SuspicionMult int `toml:"suspicion-mult"` + // PushPullInterval is the interval between complete state syncs. + // Complete state syncs are done with a single node over TCP and are + // quite expensive relative to standard gossiped messages. Setting this + // to zero will disable state push/pull syncs completely. + // + // Setting this interval lower (more frequent) will increase convergence + // speeds across larger clusters at the expense of increased bandwidth + // usage. + PushPullInterval toml.Duration `toml:"push-pull-interval"` + // ProbeInterval and ProbeTimeout are used to configure probing behavior + // for memberlist. + // + // ProbeInterval is the interval between random node probes. Setting + // this lower (more frequent) will cause the memberlist cluster to detect + // failed nodes more quickly at the expense of increased bandwidth usage. + // + // ProbeTimeout is the timeout to wait for an ack from a probed node + // before assuming it is unhealthy. This should be set to 99-percentile + // of RTT (round-trip time) on your network. + ProbeInterval toml.Duration `toml:"probe-interval"` + ProbeTimeout toml.Duration `toml:"probe-timeout"` + + // Interval and Nodes are used to configure the gossip + // behavior of memberlist. + // + // Interval is the interval between sending messages that need + // to be gossiped that haven't been able to piggyback on probing messages. + // If this is set to zero, non-piggyback gossip is disabled. By lowering + // this value (more frequent) gossip messages are propagated across + // the cluster more quickly at the expense of increased bandwidth. + // + // Nodes is the number of random nodes to send gossip messages to + // per Interval. Increasing this number causes the gossip messages + // to propagate across the cluster more quickly at the expense of + // increased bandwidth. + // + // ToTheDeadTime is the interval after which a node has died that + // we will still try to gossip to it. This gives it a chance to refute. + Interval toml.Duration `toml:"interval"` + Nodes int `toml:"nodes"` + ToTheDeadTime toml.Duration `toml:"to-the-dead-time"` +} diff --git a/pilosa.go b/pilosa.go index 2d98b910a..62eb88113 100644 --- a/pilosa.go +++ b/pilosa.go @@ -68,10 +68,6 @@ var ( ErrQueryRequired = errors.New("query required") ErrTooManyWrites = errors.New("too many write commands") - ErrConfigClusterEnabledHosts = errors.New("providing hosts to a non-disabled cluster is not allowed") - ErrConfigClusterTypeInvalid = errors.New("invalid cluster type") - ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts") - ErrClusterDoesNotOwnSlice = errors.New("cluster does not own slice") ErrNodeIDNotExists = errors.New("node with provided ID does not exist") diff --git a/server.go b/server.go index f28ca3864..6d25e3320 100644 --- a/server.go +++ b/server.go @@ -17,7 +17,6 @@ package pilosa import ( "context" "crypto/tls" - "errors" "fmt" "net" "net/http" @@ -31,6 +30,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -45,6 +45,16 @@ var _ Broadcaster = &Server{} var _ BroadcastHandler = &Server{} var _ StatusHandler = &Server{} +// ServerOption is a functional option type for pilosa.Server +type ServerOption func(s *Server) error + +func OptServerLogger(l Logger) ServerOption { + return func(s *Server) error { + s.Logger = l + return nil + } +} + // Server represents a holder wrapped by a running HTTP server. type Server struct { ln net.Listener @@ -90,7 +100,7 @@ type Server struct { } // NewServer returns a new instance of Server. -func NewServer() *Server { +func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ closing: make(chan struct{}), @@ -107,16 +117,23 @@ func NewServer() *Server { NewAttrStore: NewNopAttrStore, - AntiEntropyInterval: time.Duration(NewConfig().AntiEntropy.Interval), + AntiEntropyInterval: time.Minute * 10, MetricInterval: 0, DiagnosticInterval: 0, Logger: NopLogger, } + for _, opt := range opts { + err := opt(s) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + s.Handler.API = NewAPI() s.Handler.API.Holder = s.Holder - return s + return s, nil } // Open opens and initializes the server. diff --git a/server/cluster_test.go b/server/cluster_test.go index 7f4886612..5b0321285 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -55,7 +55,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m0.Server.Cluster.Coordinator = m0.Server.NodeID m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.Logger) - gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Config, m0.Server) + gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Server.URI.Host(), m0.Config.Gossip, m0.Server) if err != nil { t.Fatal(err) } @@ -82,7 +82,7 @@ func TestMain_SendReceiveMessage(t *testing.T) { m1.Server.Cluster.Coordinator = m0.Server.NodeID m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.Logger) - gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Config, m1.Server) + gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Server.URI.Host(), m1.Config.Gossip, m1.Server) if err != nil { t.Fatal(err) } diff --git a/server/config.go b/server/config.go new file mode 100644 index 000000000..2e2feeeb2 --- /dev/null +++ b/server/config.go @@ -0,0 +1,132 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "time" + + "github.com/pilosa/pilosa/gossip" + "github.com/pilosa/pilosa/toml" +) + +// Cluster types. +const ( + ClusterNone = "" + ClusterStatic = "static" + ClusterGossip = "gossip" +) + +// TLSConfig contains TLS configuration +type TLSConfig struct { + // CertificatePath contains the path to the certificate (.crt or .pem file) + CertificatePath string `toml:"certificate-path"` + // CertificateKeyPath contains the path to the certificate key (.key file) + CertificateKeyPath string `toml:"certificate-key-path"` + // SkipVerify disables verification for self-signed certificates + SkipVerify bool `toml:"skip-verify"` +} + +// Config represents the configuration for the command. +type Config struct { + // DataDir is the directory where Pilosa stores both indexed data and + // running state such as cluster topology information. + DataDir string `toml:"data-dir"` + // Bind is the host:port on which Pilosa will listen. + Bind string `toml:"bind"` + + // MaxWritesPerRequest limits the number of mutating commands that can be in + // a single request to the server. This includes SetBit, ClearBit, + // SetRowAttrs & SetColumnAttrs. + MaxWritesPerRequest int `toml:"max-writes-per-request"` + + // LogPath configures where Pilosa will write logs. + LogPath string `toml:"log-path"` + + // Verbose toggles verbose logging which can be useful for debugging. + Verbose bool `toml:"verbose"` + + // TLS + TLS TLSConfig + + Cluster struct { + // Disabled controls whether clustering functionality is enabled. + Disabled bool `toml:"disabled"` + Coordinator bool `toml:"coordinator"` + ReplicaN int `toml:"replicas"` + Hosts []string `toml:"hosts"` + LongQueryTime toml.Duration `toml:"long-query-time"` + } `toml:"cluster"` + + // Gossip config is based around memberlist.Config. + Gossip gossip.Config `toml:"gossip"` + + AntiEntropy struct { + Interval toml.Duration `toml:"interval"` + } `toml:"anti-entropy"` + + Metric struct { + // Service can be statsd, expvar, or none. + Service string `toml:"service"` + // Host tells the statsd client where to write. + Host string `toml:"host"` + PollInterval toml.Duration `toml:"poll-interval"` + // Diagnostics toggles sending some limited diagnostic information to + // Pilosa's developers. + Diagnostics bool `toml:"diagnostics"` + } `toml:"metric"` +} + +// NewConfig returns an instance of Config with default options. +func NewConfig() *Config { + c := &Config{ + DataDir: "~/.pilosa", + Bind: ":10101", + MaxWritesPerRequest: 5000, + // LogPath: "", + // Verbose: false, + TLS: TLSConfig{}, + } + + // Cluster config. + c.Cluster.Disabled = false + // c.Cluster.Coordinator = false + c.Cluster.ReplicaN = 1 + c.Cluster.Hosts = []string{} + c.Cluster.LongQueryTime = toml.Duration(time.Minute) + + // Gossip config. + c.Gossip.Port = "14000" + // c.Gossip.Seeds = []string{} + // c.Gossip.Key = "" + c.Gossip.StreamTimeout = toml.Duration(10 * time.Second) + c.Gossip.SuspicionMult = 4 + c.Gossip.PushPullInterval = toml.Duration(30 * time.Second) + c.Gossip.ProbeInterval = toml.Duration(1 * time.Second) + c.Gossip.ProbeTimeout = toml.Duration(500 * time.Millisecond) + c.Gossip.Interval = toml.Duration(200 * time.Millisecond) + c.Gossip.Nodes = 3 + c.Gossip.ToTheDeadTime = toml.Duration(30 * time.Second) + + // AntiEntropy config. + c.AntiEntropy.Interval = toml.Duration(10 * time.Minute) + + // Metric config. + c.Metric.Service = "none" + // c.Metric.Host = "" + c.Metric.PollInterval = toml.Duration(0 * time.Minute) + c.Metric.Diagnostics = true + + return c +} diff --git a/config_test.go b/server/config_test.go similarity index 78% rename from config_test.go rename to server/config_test.go index c07486ddb..2027ef35a 100644 --- a/config_test.go +++ b/server/config_test.go @@ -12,34 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package server_test import ( "reflect" "testing" "time" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/toml" ) func Test_NewConfig(t *testing.T) { - c := pilosa.NewConfig() + c := server.NewConfig() if c.Cluster.Disabled { t.Fatalf("unexpected Cluster.Disabled: %v", c.Cluster.Disabled) } - - // Ensure that hosts can't be specificed on a non-disabled cluster. - c.Cluster.Hosts = []string{c.Bind, "localhost:10102"} - - // Change cluster type from the default (gossip) to an invalid string. - if err := c.Validate(); err != pilosa.ErrConfigClusterEnabledHosts { - t.Fatal(err) - } } func TestDuration(t *testing.T) { - d := pilosa.Duration(time.Second * 182) + d := toml.Duration(time.Second * 182) if d.String() != "3m2s" { t.Fatalf("Unexpected time Duration %s", d) } diff --git a/server/server.go b/server/server.go index 7f66d31ed..8f4ab6147 100644 --- a/server/server.go +++ b/server/server.go @@ -56,7 +56,7 @@ type Command struct { Server *pilosa.Server // Configuration. - Config *pilosa.Config + Config *Config // Profiling options. CPUProfile string @@ -80,9 +80,10 @@ type Command struct { // NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { + s, _ := pilosa.NewServer() return &Command{ - Server: pilosa.NewServer(), - Config: pilosa.NewConfig(), + Server: s, + Config: NewConfig(), CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), @@ -150,11 +151,6 @@ func (m *Command) SetupLogger() error { // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { - err := m.Config.Validate() - if err != nil { - return err - } - m.Server.Handler.Logger = m.Server.Logger m.Server.Holder.Logger = m.Server.Logger m.Server.Holder.Stats.SetLogger(m.Server.Logger) @@ -278,7 +274,7 @@ func (m *Command) SetupNetworking() error { } m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.Logger) - gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Config, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport)) + gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport)) if err != nil { return err } diff --git a/server/server_test.go b/server/server_test.go index 497ffd7da..532257c13 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -30,6 +30,7 @@ import ( "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -489,8 +490,8 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { } // ParseConfig parses s into a Config. -func ParseConfig(s string) (pilosa.Config, error) { - var c pilosa.Config +func ParseConfig(s string) (server.Config, error) { + var c server.Config _, err := toml.Decode(s, &c) return c, err } diff --git a/toml/toml.go b/toml/toml.go new file mode 100644 index 000000000..5193ad787 --- /dev/null +++ b/toml/toml.go @@ -0,0 +1,30 @@ +package toml + +import "time" + +// Duration is a TOML wrapper type for time.Duration. +type Duration time.Duration + +// String returns the string representation of the duration. +func (d Duration) String() string { return time.Duration(d).String() } + +// UnmarshalText parses a TOML value into a duration value. +func (d *Duration) UnmarshalText(text []byte) error { + v, err := time.ParseDuration(string(text)) + if err != nil { + return err + } + + *d = Duration(v) + return nil +} + +// MarshalText writes duration value in text format. +func (d Duration) MarshalText() (text []byte, err error) { + return []byte(d.String()), nil +} + +// MarshalTOML write duration into valid TOML. +func (d Duration) MarshalTOML() ([]byte, error) { + return []byte(d.String()), nil +} From 23f6acc165f9e1395337a44a891ec0091caf86c1 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 19 Apr 2018 15:37:08 -0500 Subject: [PATCH 43/79] panic if NewCommand errors on NewServer --- server/server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 8f4ab6147..2606ff248 100644 --- a/server/server.go +++ b/server/server.go @@ -80,7 +80,11 @@ type Command struct { // NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { - s, _ := pilosa.NewServer() + s, err := pilosa.NewServer() + if err != nil { + panic(err) + } + return &Command{ Server: s, Config: NewConfig(), From 5177b243f3133b95fb90061627bf68799c5270c6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 19 Apr 2018 17:03:49 -0500 Subject: [PATCH 44/79] use ValCount return type (instead of SumCount, MinCount, and MaxCount) --- executor.go | 168 ++++++++++++++++++++---------------------- executor_test.go | 8 +- handler.go | 8 +- internal/public.pb.go | 142 +++++++++++++++++------------------ internal/public.proto | 6 +- 5 files changed, 160 insertions(+), 172 deletions(-) diff --git a/executor.go b/executor.go index 83f35e7d3..ac163cf5c 100644 --- a/executor.go +++ b/executor.go @@ -205,15 +205,15 @@ func (e *Executor) validateCallArgs(c *pql.Call) error { } // executeSum executes a Sum() call. -func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (SumCount, error) { +func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame, _ := c.Args["frame"]; frame == "" { - return SumCount{}, errors.New("Sum(): frame required") + return ValCount{}, errors.New("Sum(): frame required") } else if field, _ := c.Args["field"]; field == "" { - return SumCount{}, errors.New("Sum(): field required") + return ValCount{}, errors.New("Sum(): field required") } if len(c.Children) > 1 { - return SumCount{}, errors.New("Sum() only accepts a single bitmap input") + return ValCount{}, errors.New("Sum() only accepts a single bitmap input") } // Execute calls in bulk on each remote node and merge. @@ -223,32 +223,32 @@ func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, sl // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(SumCount) - return other.Add(v.(SumCount)) + other, _ := prev.(ValCount) + return other.Add(v.(ValCount)) } result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { - return SumCount{}, err + return ValCount{}, err } - other, _ := result.(SumCount) + other, _ := result.(ValCount) if other.Count == 0 { - return SumCount{}, nil + return ValCount{}, nil } return other, nil } // executeFieldMin executes a Min() call. -func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (MinCount, error) { +func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame, _ := c.Args["frame"]; frame == "" { - return MinCount{}, errors.New("Min(): frame required") + return ValCount{}, errors.New("Min(): frame required") } else if field, _ := c.Args["field"]; field == "" { - return MinCount{}, errors.New("Min(): field required") + return ValCount{}, errors.New("Min(): field required") } if len(c.Children) > 1 { - return MinCount{}, errors.New("Min() only accepts a single bitmap input") + return ValCount{}, errors.New("Min() only accepts a single bitmap input") } // Execute calls in bulk on each remote node and merge. @@ -258,32 +258,32 @@ func (e *Executor) executeFieldMin(ctx context.Context, index string, c *pql.Cal // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(MinCount) - return other.Smaller(v.(MinCount)) + other, _ := prev.(ValCount) + return other.Smaller(v.(ValCount)) } result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { - return MinCount{}, err + return ValCount{}, err } - other, _ := result.(MinCount) + other, _ := result.(ValCount) if other.Count == 0 { - return MinCount{}, nil + return ValCount{}, nil } return other, nil } // executeFieldMax executes a Max() call. -func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (MaxCount, error) { +func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (ValCount, error) { if frame, _ := c.Args["frame"]; frame == "" { - return MaxCount{}, errors.New("Max(): frame required") + return ValCount{}, errors.New("Max(): frame required") } else if field, _ := c.Args["field"]; field == "" { - return MaxCount{}, errors.New("Max(): field required") + return ValCount{}, errors.New("Max(): field required") } if len(c.Children) > 1 { - return MaxCount{}, errors.New("Max() only accepts a single bitmap input") + return ValCount{}, errors.New("Max() only accepts a single bitmap input") } // Execute calls in bulk on each remote node and merge. @@ -293,18 +293,18 @@ func (e *Executor) executeFieldMax(ctx context.Context, index string, c *pql.Cal // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(MaxCount) - return other.Larger(v.(MaxCount)) + other, _ := prev.(ValCount) + return other.Larger(v.(ValCount)) } result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn) if err != nil { - return MaxCount{}, err + return ValCount{}, err } - other, _ := result.(MaxCount) + other, _ := result.(ValCount) if other.Count == 0 { - return MaxCount{}, nil + return ValCount{}, nil } return other, nil } @@ -395,12 +395,12 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * } // executeSumCountSlice calculates the sum and count for fields on a slice. -func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) { +func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Bitmap if len(c.Children) == 1 { bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { - return SumCount{}, err + return ValCount{}, err } filter = bm } @@ -410,36 +410,36 @@ func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pq frame := e.Holder.Frame(index, frameName) if frame == nil { - return SumCount{}, nil + return ValCount{}, nil } field := frame.Field(fieldName) if field == nil { - return SumCount{}, nil + return ValCount{}, nil } fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) if fragment == nil { - return SumCount{}, nil + return ValCount{}, nil } vsum, vcount, err := fragment.FieldSum(filter, field.BitDepth()) if err != nil { - return SumCount{}, err + return ValCount{}, err } - return SumCount{ - Sum: int64(vsum) + (int64(vcount) * field.Min), + return ValCount{ + Val: int64(vsum) + (int64(vcount) * field.Min), Count: int64(vcount), }, nil } // executeFieldMinSlice calculates the min for fields on a slice. -func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (MinCount, error) { +func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Bitmap if len(c.Children) == 1 { bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { - return MinCount{}, err + return ValCount{}, err } filter = bm } @@ -449,36 +449,36 @@ func (e *Executor) executeFieldMinSlice(ctx context.Context, index string, c *pq frame := e.Holder.Frame(index, frameName) if frame == nil { - return MinCount{}, nil + return ValCount{}, nil } field := frame.Field(fieldName) if field == nil { - return MinCount{}, nil + return ValCount{}, nil } fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) if fragment == nil { - return MinCount{}, nil + return ValCount{}, nil } fmin, fcount, err := fragment.FieldMin(filter, field.BitDepth()) if err != nil { - return MinCount{}, err + return ValCount{}, err } - return MinCount{ - Min: int64(fmin) + field.Min, + return ValCount{ + Val: int64(fmin) + field.Min, Count: int64(fcount), }, nil } // executeFieldMaxSlice calculates the max for fields on a slice. -func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (MaxCount, error) { +func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (ValCount, error) { var filter *Bitmap if len(c.Children) == 1 { bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice) if err != nil { - return MaxCount{}, err + return ValCount{}, err } filter = bm } @@ -488,25 +488,25 @@ func (e *Executor) executeFieldMaxSlice(ctx context.Context, index string, c *pq frame := e.Holder.Frame(index, frameName) if frame == nil { - return MaxCount{}, nil + return ValCount{}, nil } field := frame.Field(fieldName) if field == nil { - return MaxCount{}, nil + return ValCount{}, nil } fragment := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice) if fragment == nil { - return MaxCount{}, nil + return ValCount{}, nil } fmax, fcount, err := fragment.FieldMax(filter, field.BitDepth()) if err != nil { - return MaxCount{}, err + return ValCount{}, err } - return MaxCount{ - Max: int64(fmax) + field.Min, + return ValCount{ + Val: int64(fmax) + field.Min, Count: int64(fcount), }, nil } @@ -1509,7 +1509,7 @@ func (e *Executor) remoteExec(ctx context.Context, node *Node, index string, q * switch call.Name { case "Average", "Sum": - v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil + v, err = decodeValCount(pb.Results[i].GetValCount()), nil case "TopN": v, err = decodePairs(pb.Results[i].GetPairs()), nil case "Count": @@ -1747,63 +1747,51 @@ func needsSlices(calls []*pql.Call) bool { return false } -// SumCount represents a grouping of sum & count for Sum() and Average() calls. -type SumCount struct { - Sum int64 `json:"sum"` +// ValCount represents a grouping of sum & count for Sum() and Average() calls. +type ValCount struct { + Val int64 `json:"value"` Count int64 `json:"count"` } -func (sc *SumCount) Add(other SumCount) SumCount { - return SumCount{ - Sum: sc.Sum + other.Sum, - Count: sc.Count + other.Count, +func (vc *ValCount) Add(other ValCount) ValCount { + return ValCount{ + Val: vc.Val + other.Val, + Count: vc.Count + other.Count, } } -func encodeSumCount(sc SumCount) *internal.SumCount { - return &internal.SumCount{ - Sum: sc.Sum, - Count: sc.Count, +func encodeValCount(vc ValCount) *internal.ValCount { + return &internal.ValCount{ + Val: vc.Val, + Count: vc.Count, } } -func decodeSumCount(pb *internal.SumCount) SumCount { - return SumCount{ - Sum: pb.Sum, +func decodeValCount(pb *internal.ValCount) ValCount { + return ValCount{ + Val: pb.Val, Count: pb.Count, } } -// MinCount represents a grouping of min and count for Min() calls. -type MinCount struct { - Min int64 `json:"min"` - Count int64 `json:"count"` -} - -// Smaller returns the smaller of the two MinCounts. -func (mc *MinCount) Smaller(other MinCount) MinCount { - if mc.Count == 0 || (other.Min < mc.Min && other.Count > 0) { +// Smaller returns the smaller of the two ValCounts. +func (vc *ValCount) Smaller(other ValCount) ValCount { + if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { return other } - return MinCount{ - Min: mc.Min, - Count: mc.Count, + return ValCount{ + Val: vc.Val, + Count: vc.Count, } } -// MaxCount represents a grouping of max and count for Max() calls. -type MaxCount struct { - Max int64 `json:"max"` - Count int64 `json:"count"` -} - -// Larger returns the larger of the two MaxCounts. -func (mc *MaxCount) Larger(other MaxCount) MaxCount { - if mc.Count == 0 || (other.Max > mc.Max && other.Count > 0) { +// Larger returns the larger of the two ValCounts. +func (vc *ValCount) Larger(other ValCount) ValCount { + if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) { return other } - return MaxCount{ - Max: mc.Max, - Count: mc.Count, + return ValCount{ + Val: vc.Val, + Count: vc.Count, } } diff --git a/executor_test.go b/executor_test.go index d5fa6e5d6..181a1a438 100644 --- a/executor_test.go +++ b/executor_test.go @@ -658,7 +658,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.MinCount{Min: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -684,7 +684,7 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } if result, err := e.Execute(context.Background(), "i", test.MustParse(pql), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.MaxCount{Max: tt.exp, Count: tt.cnt}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } } @@ -738,7 +738,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("NoFilter", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 200, Count: 5}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 200, Count: 5}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -746,7 +746,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { t.Run("WithFilter", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, row=0), frame=f, field=foo)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result[0], pilosa.SumCount{Sum: 80, Count: 2}) { + } else if !reflect.DeepEqual(result[0], pilosa.ValCount{Val: 80, Count: 2}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) diff --git a/handler.go b/handler.go index d7536f738..919008d57 100644 --- a/handler.go +++ b/handler.go @@ -1372,7 +1372,7 @@ const ( QueryResultTypeNil uint32 = iota QueryResultTypeBitmap QueryResultTypePairs - QueryResultTypeSumCount + QueryResultTypeValCount QueryResultTypeUint64 QueryResultTypeBool ) @@ -1461,9 +1461,9 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { case []Pair: pb.Results[i].Type = QueryResultTypePairs pb.Results[i].Pairs = encodePairs(result) - case SumCount: - pb.Results[i].Type = QueryResultTypeSumCount - pb.Results[i].SumCount = encodeSumCount(result) + case ValCount: + pb.Results[i].Type = QueryResultTypeValCount + pb.Results[i].ValCount = encodeValCount(result) case uint64: pb.Results[i].Type = QueryResultTypeUint64 pb.Results[i].N = result diff --git a/internal/public.pb.go b/internal/public.pb.go index dfb9a819e..31447b9f0 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -11,7 +11,7 @@ It has these top-level messages: Bitmap Pair - SumCount + ValCount Bit ColumnAttrSet Attr @@ -105,24 +105,24 @@ func (m *Pair) GetCount() uint64 { return 0 } -type SumCount struct { - Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"` +type ValCount struct { + Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"` Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *SumCount) Reset() { *m = SumCount{} } -func (m *SumCount) String() string { return proto.CompactTextString(m) } -func (*SumCount) ProtoMessage() {} -func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (m *ValCount) Reset() { *m = ValCount{} } +func (m *ValCount) String() string { return proto.CompactTextString(m) } +func (*ValCount) ProtoMessage() {} +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } -func (m *SumCount) GetSum() int64 { +func (m *ValCount) GetVal() int64 { if m != nil { - return m.Sum + return m.Val } return 0 } -func (m *SumCount) GetCount() int64 { +func (m *ValCount) GetCount() int64 { if m != nil { return m.Count } @@ -358,7 +358,7 @@ type QueryResult struct { Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } @@ -395,9 +395,9 @@ func (m *QueryResult) GetPairs() []*Pair { return nil } -func (m *QueryResult) GetSumCount() *SumCount { +func (m *QueryResult) GetValCount() *ValCount { if m != nil { - return m.SumCount + return m.ValCount } return nil } @@ -548,7 +548,7 @@ func (m *ImportValueRequest) GetValues() []int64 { func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") - proto.RegisterType((*SumCount)(nil), "internal.SumCount") + proto.RegisterType((*ValCount)(nil), "internal.ValCount") proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") proto.RegisterType((*Attr)(nil), "internal.Attr") @@ -655,7 +655,7 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *SumCount) Marshal() (dAtA []byte, err error) { +func (m *ValCount) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -665,15 +665,15 @@ func (m *SumCount) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *SumCount) MarshalTo(dAtA []byte) (int, error) { +func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int _ = l - if m.Sum != 0 { + if m.Val != 0 { dAtA[i] = 0x8 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Sum)) + i = encodeVarintPublic(dAtA, i, uint64(m.Val)) } if m.Count != 0 { dAtA[i] = 0x10 @@ -1023,11 +1023,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.SumCount != nil { + if m.ValCount != nil { dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size())) - n6, err := m.SumCount.MarshalTo(dAtA[i:]) + i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) + n6, err := m.ValCount.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -1317,11 +1317,11 @@ func (m *Pair) Size() (n int) { return n } -func (m *SumCount) Size() (n int) { +func (m *ValCount) Size() (n int) { var l int _ = l - if m.Sum != 0 { - n += 1 + sovPublic(uint64(m.Sum)) + if m.Val != 0 { + n += 1 + sovPublic(uint64(m.Val)) } if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) @@ -1471,8 +1471,8 @@ func (m *QueryResult) Size() (n int) { if m.Changed { n += 2 } - if m.SumCount != nil { - l = m.SumCount.Size() + if m.ValCount != nil { + l = m.ValCount.Size() n += 1 + l + sovPublic(uint64(l)) } if m.Type != 0 { @@ -1874,7 +1874,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } -func (m *SumCount) Unmarshal(dAtA []byte) error { +func (m *ValCount) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1897,17 +1897,17 @@ func (m *SumCount) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: SumCount: wiretype end group for non-group") + return fmt.Errorf("proto: ValCount: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ValCount: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Val", wireType) } - m.Sum = 0 + m.Val = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1917,7 +1917,7 @@ func (m *SumCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Sum |= (int64(b) & 0x7F) << shift + m.Val |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -2959,7 +2959,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { m.Changed = bool(v != 0) case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ValCount", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -2983,10 +2983,10 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.SumCount == nil { - m.SumCount = &SumCount{} + if m.ValCount == nil { + m.ValCount = &ValCount{} } - if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.ValCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -3821,47 +3821,47 @@ var fileDescriptorPublic = []byte{ // 705 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x40, 0x14, 0x65, 0x62, 0x27, 0x71, 0x6e, 0x92, 0xaa, 0x1a, 0x41, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, - 0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x29, 0x65, 0xed, 0xb6, 0xa3, + 0x57, 0xa9, 0x14, 0xf6, 0x20, 0xd2, 0x87, 0x14, 0x55, 0x54, 0x30, 0x2d, 0x61, 0xed, 0xb6, 0xa3, 0x62, 0xc9, 0x2f, 0xec, 0xb1, 0xda, 0x7c, 0x07, 0x1b, 0x3e, 0x81, 0x8f, 0x60, 0xc5, 0x0a, 0x76, 0x7c, 0x02, 0x94, 0x1f, 0x41, 0xf7, 0x8e, 0x27, 0x76, 0x5a, 0x09, 0x58, 0xb0, 0x9b, 0x73, 0xce, 0xcc, 0xf5, 0x9c, 0xb9, 0xe7, 0x26, 0x30, 0xca, 0xab, 0xb3, 0x38, 0x3a, 0x9f, 0xe6, 0x45, 0xa6, - 0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x29, 0xf4, 0xe6, 0x91, 0x4a, 0xc2, 0x9c, - 0x73, 0xb0, 0xe7, 0x91, 0x2a, 0x5d, 0xe6, 0x59, 0x81, 0x2d, 0x68, 0xcd, 0x9f, 0x40, 0xf7, 0x85, - 0x52, 0x45, 0xe9, 0x76, 0x3c, 0x2b, 0x18, 0xce, 0xb6, 0xa6, 0xe6, 0xdc, 0x14, 0x69, 0xa1, 0x45, - 0x3c, 0x79, 0x24, 0x57, 0xa5, 0x6b, 0x79, 0x56, 0x30, 0x10, 0xb4, 0xf6, 0x9f, 0x81, 0xfd, 0x2a, - 0x8c, 0x0a, 0xbe, 0x05, 0x9d, 0xc5, 0xbe, 0xcb, 0x3c, 0x16, 0xd8, 0xa2, 0xb3, 0xd8, 0xe7, 0xf7, - 0xa1, 0xbb, 0x97, 0x55, 0xa9, 0x72, 0x3b, 0x44, 0x69, 0xc0, 0xb7, 0xc1, 0x3a, 0x92, 0x2b, 0xd7, - 0xf2, 0x58, 0x30, 0x10, 0xb8, 0xf4, 0x67, 0xe0, 0x2c, 0xab, 0x64, 0xad, 0x2e, 0xab, 0x84, 0x8a, - 0x58, 0x02, 0x97, 0x9b, 0x55, 0xac, 0xba, 0x8a, 0xff, 0x06, 0xac, 0x79, 0xa4, 0x50, 0x14, 0xd9, - 0xd5, 0xfa, 0xab, 0x1a, 0xf0, 0x47, 0xe0, 0xec, 0x65, 0x71, 0x95, 0xa4, 0x8b, 0xfd, 0xfa, 0xdb, - 0x6b, 0xcc, 0x1f, 0xc3, 0xe0, 0x24, 0x4a, 0x64, 0xa9, 0xc2, 0x24, 0xa7, 0x4b, 0x58, 0xa2, 0x21, - 0xfc, 0xb7, 0x30, 0xd6, 0x3b, 0xd1, 0xed, 0x52, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a, - 0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xac, 0x72, 0x59, 0xdf, 0x94, - 0xd6, 0xdc, 0x83, 0xe1, 0x52, 0x15, 0x51, 0x7a, 0x79, 0x1a, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29, + 0x32, 0xee, 0x44, 0xa9, 0x92, 0x45, 0x1a, 0xc6, 0xfe, 0x12, 0x7a, 0xf3, 0x48, 0x25, 0x61, 0xce, + 0x39, 0xd8, 0xf3, 0x48, 0x95, 0x2e, 0xf3, 0xac, 0xc0, 0x16, 0xb4, 0xe6, 0x4f, 0xa0, 0xfb, 0x42, + 0xa9, 0xa2, 0x74, 0x3b, 0x9e, 0x15, 0x0c, 0x67, 0x5b, 0x53, 0x73, 0x6e, 0x8a, 0xb4, 0xd0, 0x22, + 0x9e, 0x3c, 0x92, 0xab, 0xd2, 0xb5, 0x3c, 0x2b, 0x18, 0x08, 0x5a, 0xfb, 0xcf, 0xc0, 0x7e, 0x15, + 0x46, 0x05, 0xdf, 0x82, 0xce, 0x62, 0xdf, 0x65, 0x1e, 0x0b, 0x6c, 0xd1, 0x59, 0xec, 0xf3, 0xfb, + 0xd0, 0xdd, 0xcb, 0xaa, 0x54, 0xb9, 0x1d, 0xa2, 0x34, 0xe0, 0xdb, 0x60, 0x1d, 0xc9, 0x95, 0x6b, + 0x79, 0x2c, 0x18, 0x08, 0x5c, 0xfa, 0x33, 0x70, 0x96, 0x61, 0xbc, 0x56, 0x97, 0x61, 0x4c, 0x45, + 0x2c, 0x81, 0xcb, 0xcd, 0x2a, 0x56, 0x5d, 0xc5, 0x7f, 0x03, 0xd6, 0x3c, 0x52, 0x28, 0x8a, 0xec, + 0x6a, 0xfd, 0x55, 0x0d, 0xf8, 0x23, 0x70, 0xf6, 0xb2, 0xb8, 0x4a, 0xd2, 0xc5, 0x7e, 0xfd, 0xed, + 0x35, 0xe6, 0x8f, 0x61, 0x70, 0x1a, 0x25, 0xb2, 0x54, 0x61, 0x92, 0xd3, 0x25, 0x2c, 0xd1, 0x10, + 0xfe, 0x5b, 0x18, 0xeb, 0x9d, 0xe8, 0xf6, 0x44, 0xaa, 0x3b, 0x9e, 0xfe, 0xed, 0x95, 0xee, 0x7a, + 0xfc, 0xc4, 0xc0, 0x46, 0xcd, 0x48, 0x6c, 0x2d, 0xe1, 0x93, 0x9e, 0xae, 0x72, 0x59, 0xdf, 0x94, + 0xd6, 0xdc, 0x83, 0xe1, 0x89, 0x2a, 0xa2, 0xf4, 0x72, 0x19, 0xc6, 0x95, 0xac, 0x0b, 0xb5, 0x29, 0xf4, 0xb8, 0x48, 0x95, 0x96, 0x6d, 0xb2, 0xb1, 0xc6, 0xe8, 0x71, 0x9e, 0x65, 0xb1, 0x16, 0xbb, 0x1e, 0x0b, 0x1c, 0xd1, 0x10, 0x7c, 0x02, 0x70, 0x18, 0x67, 0x61, 0x7d, 0xb6, 0xe7, 0xb1, 0x80, 0x89, 0x16, 0xe3, 0xef, 0x42, 0x1f, 0x6f, 0xfa, 0x32, 0xcc, 0x1b, 0xb7, 0xec, 0x0f, 0x6e, 0xfd, 0xcf, 0x0c, 0x46, 0xaf, 0x2b, 0x59, 0xac, 0x84, 0x7c, 0x5f, 0xc9, 0x92, 0xba, 0x42, 0xb8, 0x76, - 0xa9, 0x01, 0xdf, 0x81, 0xde, 0x32, 0x8e, 0xce, 0xa5, 0x7e, 0x3b, 0x5b, 0xd4, 0x08, 0xbd, 0x36, - 0x6f, 0x5e, 0x92, 0x57, 0x47, 0xb4, 0x29, 0x3c, 0x29, 0x64, 0x92, 0x29, 0x63, 0xa6, 0x46, 0xdc, - 0x87, 0xd1, 0xc1, 0xf5, 0x79, 0x5c, 0x5d, 0x48, 0x7d, 0xb4, 0x47, 0xea, 0x06, 0x87, 0xd5, 0x6b, - 0x4c, 0x89, 0xef, 0xeb, 0xea, 0x2d, 0xca, 0xff, 0xc0, 0x60, 0x5c, 0x5f, 0xbf, 0xcc, 0xb3, 0xb4, - 0x94, 0xd8, 0xa3, 0x83, 0xa2, 0x30, 0x3d, 0x3a, 0x28, 0x0a, 0xbe, 0x0b, 0x7d, 0x21, 0xcb, 0x2a, - 0x56, 0xa6, 0xf1, 0x0f, 0x9a, 0xa7, 0x30, 0x67, 0xab, 0x58, 0x09, 0xb3, 0x8b, 0x3f, 0x87, 0xad, - 0x8d, 0x20, 0xe9, 0x89, 0x19, 0xce, 0x1e, 0x36, 0xe7, 0x36, 0x74, 0x71, 0x6b, 0xbb, 0xff, 0x8d, - 0xc1, 0xb0, 0x55, 0x99, 0x07, 0x66, 0x78, 0xe9, 0x5a, 0xc3, 0xd9, 0x76, 0x53, 0x48, 0xf3, 0xc2, - 0x0c, 0xf7, 0x08, 0xd8, 0x71, 0x1d, 0x26, 0x76, 0x8c, 0x2d, 0xc4, 0xe1, 0x34, 0xdf, 0x6f, 0xb5, - 0x10, 0x69, 0xa1, 0x45, 0xee, 0x42, 0x7f, 0xef, 0x5d, 0x98, 0x5e, 0xca, 0x0b, 0x0a, 0x93, 0x23, - 0x0c, 0xe4, 0xd3, 0x66, 0x38, 0xe9, 0xf5, 0x87, 0x33, 0xde, 0x94, 0x30, 0x8a, 0x68, 0x06, 0xd8, - 0xa4, 0x19, 0x7b, 0x31, 0xd6, 0x69, 0xf6, 0x7f, 0x32, 0x18, 0x2f, 0x92, 0x3c, 0x2b, 0x54, 0x2b, - 0x21, 0x8b, 0xf4, 0x42, 0x5e, 0x9b, 0x84, 0x10, 0x40, 0xf6, 0xb0, 0x08, 0x13, 0x3d, 0x0a, 0x03, - 0xa1, 0x01, 0xb2, 0x94, 0x14, 0x4a, 0x86, 0x2d, 0x34, 0xa0, 0x4c, 0xe0, 0xb0, 0x97, 0xae, 0xad, - 0xd3, 0xa4, 0x11, 0x66, 0xdf, 0xcc, 0x7a, 0xe9, 0x76, 0x49, 0x6a, 0x08, 0xcc, 0xfe, 0x7a, 0xd8, - 0x31, 0x2f, 0x56, 0x60, 0x89, 0x16, 0x83, 0xef, 0x20, 0xb2, 0x2b, 0xfa, 0x85, 0xeb, 0xd3, 0x2f, - 0x9c, 0x81, 0x78, 0x52, 0x97, 0x21, 0xd1, 0x21, 0xb1, 0xc5, 0xf8, 0x5f, 0x18, 0x70, 0xed, 0x91, - 0xa6, 0xe8, 0xff, 0x19, 0xc5, 0xbd, 0x91, 0x8c, 0x75, 0x63, 0x70, 0x2f, 0x82, 0xbf, 0xd8, 0xdc, - 0x81, 0x1e, 0xdd, 0xc2, 0x58, 0xac, 0xd1, 0x2d, 0x13, 0xfd, 0xdb, 0x26, 0xe6, 0xdb, 0x5f, 0x6f, - 0x26, 0xec, 0xfb, 0xcd, 0x84, 0xfd, 0xb8, 0x99, 0xb0, 0x8f, 0xbf, 0x26, 0xf7, 0xce, 0x7a, 0xf4, - 0x27, 0xf2, 0xf4, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa3, 0xa0, 0xd2, 0x51, 0x54, 0x06, 0x00, + 0xa9, 0x01, 0xdf, 0x81, 0xde, 0x49, 0x1c, 0x9d, 0x4b, 0xfd, 0x76, 0xb6, 0xa8, 0x11, 0x7a, 0x6d, + 0xde, 0xbc, 0x24, 0xaf, 0x8e, 0x68, 0x53, 0x78, 0x52, 0xc8, 0x24, 0x53, 0xc6, 0x4c, 0x8d, 0xb8, + 0x0f, 0xa3, 0x83, 0xeb, 0xf3, 0xb8, 0xba, 0x90, 0xfa, 0x68, 0x8f, 0xd4, 0x0d, 0x0e, 0xab, 0xd7, + 0x98, 0x12, 0xdf, 0xd7, 0xd5, 0x5b, 0x94, 0xff, 0x81, 0xc1, 0xb8, 0xbe, 0x7e, 0x99, 0x67, 0x69, + 0x29, 0xb1, 0x47, 0x07, 0x45, 0x61, 0x7a, 0x74, 0x50, 0x14, 0x7c, 0x17, 0xfa, 0x42, 0x96, 0x55, + 0xac, 0x4c, 0xe3, 0x1f, 0x34, 0x4f, 0x61, 0xce, 0x56, 0xb1, 0x12, 0x66, 0x17, 0x7f, 0x0e, 0x5b, + 0x1b, 0x41, 0xd2, 0x13, 0x33, 0x9c, 0x3d, 0x6c, 0xce, 0x6d, 0xe8, 0xe2, 0xd6, 0x76, 0xff, 0x1b, + 0x83, 0x61, 0xab, 0x32, 0x0f, 0xcc, 0xf0, 0xd2, 0xb5, 0x86, 0xb3, 0xed, 0xa6, 0x90, 0xe6, 0x85, + 0x19, 0xee, 0x11, 0xb0, 0xe3, 0x3a, 0x4c, 0xec, 0x18, 0x5b, 0x88, 0xc3, 0x69, 0xbe, 0xdf, 0x6a, + 0x21, 0xd2, 0x42, 0x8b, 0xdc, 0x85, 0xfe, 0xde, 0xbb, 0x30, 0xbd, 0x94, 0x17, 0x14, 0x26, 0x47, + 0x18, 0xc8, 0xa7, 0xcd, 0x70, 0xd2, 0xeb, 0x0f, 0x67, 0xbc, 0x29, 0x61, 0x14, 0xd1, 0x0c, 0xb0, + 0x49, 0x33, 0xf6, 0x62, 0xac, 0xd3, 0xec, 0xff, 0x64, 0x30, 0x5e, 0x24, 0x79, 0x56, 0xa8, 0x56, + 0x42, 0x16, 0xe9, 0x85, 0xbc, 0x36, 0x09, 0x21, 0x80, 0xec, 0x61, 0x11, 0x26, 0x7a, 0x14, 0x06, + 0x42, 0x03, 0x64, 0x29, 0x29, 0x94, 0x0c, 0x5b, 0x68, 0x40, 0x99, 0xc0, 0x61, 0x2f, 0x5d, 0x5b, + 0xa7, 0x49, 0x23, 0xcc, 0xbe, 0x99, 0xf5, 0xd2, 0xed, 0x92, 0xd4, 0x10, 0x98, 0xfd, 0xf5, 0xb0, + 0x63, 0x5e, 0xac, 0xc0, 0x12, 0x2d, 0x06, 0xdf, 0x41, 0x64, 0x57, 0xf4, 0x0b, 0xd7, 0xa7, 0x5f, + 0x38, 0x03, 0xf1, 0xa4, 0x2e, 0x43, 0xa2, 0x43, 0x62, 0x8b, 0xf1, 0xbf, 0x30, 0xe0, 0xda, 0x23, + 0x4d, 0xd1, 0xff, 0x33, 0x8a, 0x7b, 0x23, 0x19, 0xeb, 0xc6, 0xe0, 0x5e, 0x04, 0x7f, 0xb1, 0xb9, + 0x03, 0x3d, 0xba, 0x85, 0xb1, 0x58, 0xa3, 0x5b, 0x26, 0xfa, 0xb7, 0x4d, 0xcc, 0xb7, 0xbf, 0xde, + 0x4c, 0xd8, 0xf7, 0x9b, 0x09, 0xfb, 0x71, 0x33, 0x61, 0x1f, 0x7f, 0x4d, 0xee, 0x9d, 0xf5, 0xe8, + 0x4f, 0xe4, 0xe9, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x3e, 0xba, 0x23, 0x54, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 5fa40cfab..026a3748b 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -14,8 +14,8 @@ message Pair { uint64 Count = 2; } -message SumCount { - int64 Sum = 1; +message ValCount { + int64 Val = 1; int64 Count = 2; } @@ -64,7 +64,7 @@ message QueryResult { Bitmap Bitmap = 1; uint64 N = 2; repeated Pair Pairs = 3; - SumCount SumCount = 5; + ValCount ValCount = 5; bool Changed = 4; } From edee152a9bc06756fc2a32180f4a1f981f868b52 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 20 Apr 2018 13:20:56 -0500 Subject: [PATCH 45/79] remove Index.MergeSchemas() method --- index.go | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/index.go b/index.go index 4676e1b43..00ee98f11 100644 --- a/index.go +++ b/index.go @@ -481,46 +481,6 @@ func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p indexInfoSlice) Len() int { return len(p) } func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } -// MergeSchemas combines indexes and frames from a and b into one schema. -func MergeSchemas(a, b []*IndexInfo) []*IndexInfo { - // Generate a map from both schemas. - m := make(map[string]map[string]map[string]struct{}) - for _, idxs := range [][]*IndexInfo{a, b} { - for _, idx := range idxs { - if m[idx.Name] == nil { - m[idx.Name] = make(map[string]map[string]struct{}) - } - for _, frame := range idx.Frames { - if m[idx.Name][frame.Name] == nil { - m[idx.Name][frame.Name] = make(map[string]struct{}) - } - for _, view := range frame.Views { - m[idx.Name][frame.Name][view.Name] = struct{}{} - } - } - } - } - - // Generate new schema from map. - idxs := make([]*IndexInfo, 0, len(m)) - for idx, frames := range m { - di := &IndexInfo{Name: idx} - for frame, views := range frames { - fi := &FrameInfo{Name: frame} - for view := range views { - fi.Views = append(fi.Views, &ViewInfo{Name: view}) - } - sort.Sort(viewInfoSlice(fi.Views)) - di.Frames = append(di.Frames, fi) - } - sort.Sort(frameInfoSlice(di.Frames)) - idxs = append(idxs, di) - } - sort.Sort(indexInfoSlice(idxs)) - - return idxs -} - // EncodeIndexes converts a into its internal representation. func EncodeIndexes(a []*Index) []*internal.Index { other := make([]*internal.Index, len(a)) From f2729b90d7ba5e033a8e2aa13ac8804bf0d73379 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 10:43:06 -0500 Subject: [PATCH 46/79] vendor github.com/golang/groupcache/lru. rebuild Gopkg.lock --- Gopkg.lock | 58 +++++++++++++++++------ NOTICE | 36 +++++++++++++++ cache.go | 2 +- lru/lru.go | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 214 insertions(+), 15 deletions(-) create mode 100644 NOTICE create mode 100644 lru/lru.go diff --git a/Gopkg.lock b/Gopkg.lock index af6174c08..f29660b22 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -57,7 +57,10 @@ [[projects]] name = "github.com/go-ole/go-ole" - packages = [".","oleutil"] + packages = [ + ".", + "oleutil" + ] revision = "0e87ea779d9deb219633b828a023b32e1244dd57" version = "v1.2.0" @@ -67,12 +70,6 @@ revision = "100ba4e885062801d56799d78530b73b178a78f3" version = "v0.4" -[[projects]] - branch = "master" - name = "github.com/golang/groupcache" - packages = ["lru"] - revision = "84a468cf14b4376def5d68c722b139b881c450a4" - [[projects]] branch = "master" name = "github.com/golang/protobuf" @@ -130,7 +127,17 @@ [[projects]] branch = "master" name = "github.com/hashicorp/hcl" - packages = [".","hcl/ast","hcl/parser","hcl/scanner","hcl/strconv","hcl/token","json/parser","json/scanner","json/token"] + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token" + ] revision = "23c074d0eceb2b8a5bfdbb271ab780cde70f05a8" [[projects]] @@ -154,7 +161,10 @@ [[projects]] branch = "master" name = "github.com/miekg/dns" - packages = [".","internal/socket"] + packages = [ + ".", + "internal/socket" + ] revision = "9fc4eb252eedf0ef8adc05169ce35da5e31beaba" [[projects]] @@ -195,7 +205,14 @@ [[projects]] name = "github.com/shirou/gopsutil" - packages = ["cpu","host","internal/common","mem","net","process"] + packages = [ + "cpu", + "host", + "internal/common", + "mem", + "net", + "process" + ] revision = "bfe3c2e8f406bf352bc8df81f98c752224867349" version = "v2.17.11" @@ -208,7 +225,10 @@ [[projects]] branch = "master" name = "github.com/spf13/afero" - packages = [".","mem"] + packages = [ + ".", + "mem" + ] revision = "5660eeed305fe5f69c8fc6cf899132a459a97064" [[projects]] @@ -256,13 +276,23 @@ [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix","windows"] + packages = [ + "unix", + "windows" + ] revision = "1e2299c37cc91a509f1b12369872d27be0ce98a6" [[projects]] branch = "master" name = "golang.org/x/text" - packages = ["internal/gen","internal/triegen","internal/ucd","transform","unicode/cldr","unicode/norm"] + packages = [ + "internal/gen", + "internal/triegen", + "internal/ucd", + "transform", + "unicode/cldr", + "unicode/norm" + ] revision = "88f656faf3f37f690df1a32515b479415e1a6769" [[projects]] @@ -274,6 +304,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "668c03e22d947dd7e24c4c5e07d7850ed0a2ea9a27d2af1fa0d1cff0e3743b0b" + inputs-digest = "61a00007fe7398d958fc8668543d42293f1310f9f9b74c63ff96f5a892eab1e7" solver-name = "gps-cdcl" solver-version = 1 diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..7613b11e7 --- /dev/null +++ b/NOTICE @@ -0,0 +1,36 @@ +Software license +================ + +Copyright 2017 Pilosa Corp. + +Licensed under the Apache License, Version 2.0 (the "License"). +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Third-party software licenses +============================= + +The file /pilosa/lru/lru.go contains a redistribution of lru +(github.com/golang/groupcache/lru); the license follows: + + Copyright 2013 Google Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cache.go b/cache.go index 9c909ced4..e27d2f93c 100644 --- a/cache.go +++ b/cache.go @@ -22,8 +22,8 @@ import ( "sync" "time" - "github.com/golang/groupcache/lru" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/lru" ) const ( diff --git a/lru/lru.go b/lru/lru.go new file mode 100644 index 000000000..532cc45e6 --- /dev/null +++ b/lru/lru.go @@ -0,0 +1,133 @@ +/* +Copyright 2013 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package lru implements an LRU cache. +package lru + +import "container/list" + +// Cache is an LRU cache. It is not safe for concurrent access. +type Cache struct { + // MaxEntries is the maximum number of cache entries before + // an item is evicted. Zero means no limit. + MaxEntries int + + // OnEvicted optionally specificies a callback function to be + // executed when an entry is purged from the cache. + OnEvicted func(key Key, value interface{}) + + ll *list.List + cache map[interface{}]*list.Element +} + +// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators +type Key interface{} + +type entry struct { + key Key + value interface{} +} + +// New creates a new Cache. +// If maxEntries is zero, the cache has no limit and it's assumed +// that eviction is done by the caller. +func New(maxEntries int) *Cache { + return &Cache{ + MaxEntries: maxEntries, + ll: list.New(), + cache: make(map[interface{}]*list.Element), + } +} + +// Add adds a value to the cache. +func (c *Cache) Add(key Key, value interface{}) { + if c.cache == nil { + c.cache = make(map[interface{}]*list.Element) + c.ll = list.New() + } + if ee, ok := c.cache[key]; ok { + c.ll.MoveToFront(ee) + ee.Value.(*entry).value = value + return + } + ele := c.ll.PushFront(&entry{key, value}) + c.cache[key] = ele + if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries { + c.RemoveOldest() + } +} + +// Get looks up a key's value from the cache. +func (c *Cache) Get(key Key) (value interface{}, ok bool) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.ll.MoveToFront(ele) + return ele.Value.(*entry).value, true + } + return +} + +// Remove removes the provided key from the cache. +func (c *Cache) Remove(key Key) { + if c.cache == nil { + return + } + if ele, hit := c.cache[key]; hit { + c.removeElement(ele) + } +} + +// RemoveOldest removes the oldest item from the cache. +func (c *Cache) RemoveOldest() { + if c.cache == nil { + return + } + ele := c.ll.Back() + if ele != nil { + c.removeElement(ele) + } +} + +func (c *Cache) removeElement(e *list.Element) { + c.ll.Remove(e) + kv := e.Value.(*entry) + delete(c.cache, kv.key) + if c.OnEvicted != nil { + c.OnEvicted(kv.key, kv.value) + } +} + +// Len returns the number of items in the cache. +func (c *Cache) Len() int { + if c.cache == nil { + return 0 + } + return c.ll.Len() +} + +// Clear purges all stored items from the cache. +func (c *Cache) Clear() { + if c.OnEvicted != nil { + for _, e := range c.cache { + kv := e.Value.(*entry) + c.OnEvicted(kv.key, kv.value) + } + } + c.ll = nil + c.cache = nil +} From 3282cf8cbe8e9776ad81e506c7a0065c6c38ce55 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 11:32:41 -0500 Subject: [PATCH 47/79] remove PATCH frame endpoint --- api.go | 73 ++++++++++++++++--------------------------- apimethod_string.go | 4 +-- docs/api-reference.md | 29 ----------------- handler.go | 41 ------------------------ handler_test.go | 24 -------------- 5 files changed, 29 insertions(+), 142 deletions(-) diff --git a/api.go b/api.go index a5953217d..700b21d25 100644 --- a/api.go +++ b/api.go @@ -925,23 +925,6 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest return err } -// ModifyFrameTimeQuantum changes the time quantum on the given frame. TODO: -// what happens if there is already data in the frame? -func (api *API) ModifyFrameTimeQuantum(ctx context.Context, indexName string, frameName string, timeQuantum TimeQuantum) error { - if err := api.validate(apiModifyFrameTimeQuantum); err != nil { - return errors.Wrap(err, "validate api method") - } - - // Retrieve index by name. - frame := api.Holder.Frame(indexName, frameName) - if frame == nil { - return ErrFrameNotFound - } - - // Set default time quantum on index. - return frame.SetTimeQuantum(timeQuantum) -} - // MaxSlices returns the maximum slice number for each index in a map. func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxSlices() @@ -1168,7 +1151,6 @@ const ( apiMarshalFragment //apiMaxInverseSlices // not implemented //apiMaxSlices // not implemented - apiModifyFrameTimeQuantum apiQuery apiRecalculateCaches apiRemoveNode @@ -1196,32 +1178,31 @@ var methodsResizing = map[apiMethod]struct{}{ } var methodsNormal = map[apiMethod]struct{}{ - apiCreateField: struct{}{}, - apiCreateFrame: struct{}{}, - apiCreateIndex: struct{}{}, - apiCreateInputDefinition: struct{}{}, - apiDeleteField: struct{}{}, - apiDeleteFrame: struct{}{}, - apiDeleteIndex: struct{}{}, - apiDeleteInputDefinition: struct{}{}, - apiDeleteView: struct{}{}, - apiExportCSV: struct{}{}, - apiFields: struct{}{}, - apiFragmentBlockData: struct{}{}, - apiFragmentBlocks: struct{}{}, - apiFrameAttrDiff: struct{}{}, - apiImport: struct{}{}, - apiImportValue: struct{}{}, - apiIndex: struct{}{}, - apiIndexAttrDiff: struct{}{}, - apiInputDefinition: struct{}{}, - apiModifyFrameTimeQuantum: struct{}{}, - apiQuery: struct{}{}, - apiRecalculateCaches: struct{}{}, - apiRemoveNode: struct{}{}, - apiRestoreFrame: struct{}{}, - apiSliceNodes: struct{}{}, - apiUnmarshalFragment: struct{}{}, - apiViews: struct{}{}, - apiWriteInput: struct{}{}, + apiCreateField: struct{}{}, + apiCreateFrame: struct{}{}, + apiCreateIndex: struct{}{}, + apiCreateInputDefinition: struct{}{}, + apiDeleteField: struct{}{}, + apiDeleteFrame: struct{}{}, + apiDeleteIndex: struct{}{}, + apiDeleteInputDefinition: struct{}{}, + apiDeleteView: struct{}{}, + apiExportCSV: struct{}{}, + apiFields: struct{}{}, + apiFragmentBlockData: struct{}{}, + apiFragmentBlocks: struct{}{}, + apiFrameAttrDiff: struct{}{}, + apiImport: struct{}{}, + apiImportValue: struct{}{}, + apiIndex: struct{}{}, + apiIndexAttrDiff: struct{}{}, + apiInputDefinition: struct{}{}, + apiQuery: struct{}{}, + apiRecalculateCaches: struct{}{}, + apiRemoveNode: struct{}{}, + apiRestoreFrame: struct{}{}, + apiSliceNodes: struct{}{}, + apiUnmarshalFragment: struct{}{}, + apiViews: struct{}{}, + apiWriteInput: struct{}{}, } diff --git a/apimethod_string.go b/apimethod_string.go index 8f77b69e7..8ee574f8d 100644 --- a/apimethod_string.go +++ b/apimethod_string.go @@ -4,9 +4,9 @@ package pilosa import "fmt" -const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiModifyFrameTimeQuantumapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" +const _apiMethod_name = "apiClusterMessageapiCreateFieldapiCreateFrameapiCreateIndexapiCreateInputDefinitionapiDeleteFieldapiDeleteFrameapiDeleteIndexapiDeleteInputDefinitionapiDeleteViewapiExportCSVapiFieldsapiFragmentBlockDataapiFragmentBlocksapiFrameAttrDiffapiImportapiImportValueapiIndexapiIndexAttrDiffapiInputDefinitionapiMarshalFragmentapiQueryapiRecalculateCachesapiRemoveNodeapiResizeAbortapiRestoreFrameapiSetCoordinatorapiSliceNodesapiUnmarshalFragmentapiViewsapiWriteInput" -var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 344, 352, 372, 385, 399, 414, 431, 444, 464, 472, 485} +var _apiMethod_index = [...]uint16{0, 17, 31, 45, 59, 83, 97, 111, 125, 149, 162, 174, 183, 203, 220, 236, 245, 259, 267, 283, 301, 319, 327, 347, 360, 374, 389, 406, 419, 439, 447, 460} func (i apiMethod) String() string { if i < 0 || i >= apiMethod(len(_apiMethod_index)-1) { diff --git a/docs/api-reference.md b/docs/api-reference.md index f7c3641b9..3bcc33559 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -149,35 +149,6 @@ curl -XDELETE localhost:10101/index/user/frame/language {} ``` -### Change frame time quantum - -`PATCH /index//frame//time-quantum` - -Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame. - -The payload is in JSON with the format: `{"timeQuantum": "${TIME_QUANTUM}"}`. Valid time quantum values are: - -* (Empty string) -* Y: year -* M: month -* D: day -* H: hour -* YM: year and month -* MD: month and day -* DH: day and hour -* YMD: year, month and day -* MDH: month, day and hour -* YMDH: year, month, day and hour - -``` request -curl localhost:10101/index/user/frame/language/time-quantum \ - -X POST \ - -d '{"timeQuantum": "YM"}' -``` -``` response -{} -``` - ### Create Field `POST /index//frame//field/` diff --git a/handler.go b/handler.go index bd97c1d1a..da0360906 100644 --- a/handler.go +++ b/handler.go @@ -157,7 +157,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore") - router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE") @@ -596,46 +595,6 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { type deleteFrameResponse struct{} -// handlePatchFrameTimeQuantum handles PATCH /frame/time_quantum request. -func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - // Decode request. - var req patchFrameTimeQuantumRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - // Validate quantum. - tq, err := ParseTimeQuantum(req.TimeQuantum) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := h.API.ModifyFrameTimeQuantum(r.Context(), indexName, frameName, tq); err != nil { - if err == ErrFragmentNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - return - } - - // Encode response. - if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil { - h.Logger.Printf("response encoding error: %s", err) - } -} - -type patchFrameTimeQuantumRequest struct { - TimeQuantum string `json:"timeQuantum"` -} - -type patchFrameTimeQuantumResponse struct{} - // handlePostFrameField handles POST /frame/field request. func (h *Handler) handlePostFrameField(w http.ResponseWriter, r *http.Request) { indexName := mux.Vars(r)["index"] diff --git a/handler_test.go b/handler_test.go index 8c719feb1..4187b3303 100644 --- a/handler_test.go +++ b/handler_test.go @@ -748,30 +748,6 @@ func TestHandler_DeleteFrame(t *testing.T) { } } -// Ensure handler can set the frame time quantum. -func TestHandler_SetFrameTimeQuantum(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - // Create frame. - if _, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - h := test.NewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } else if q := hldr.Index("i0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { - t.Fatalf("unexpected time quantum: %s", q) - } -} - // Ensure the handler can return data in differing blocks for an index. func TestHandler_Index_AttrStore_Diff(t *testing.T) { hldr := test.MustOpenHolder() From 23f5c7166bb6a1aaabe3541050c7331ab2b8d7cb Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 13:23:35 -0500 Subject: [PATCH 48/79] cleat up flipBitmap and add tests --- roaring/roaring.go | 52 +++++++++++++++++++++-------- roaring/roaring_helpers_test.go | 5 ++- roaring/roaring_internal_test.go | 57 ++++++++++++++------------------ 3 files changed, 66 insertions(+), 48 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 0087e6fdf..2a8b6645c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1674,19 +1674,6 @@ func (c *container) clone() *container { return other } -// flipBitmap returns a new bitmap containter containing the inverse of all -// bits in c. -func (c *container) flipBitmap() *container { - other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} - - for i, bitmap := range c.bitmap { - other.bitmap[i] = ^bitmap - } - - other.n = other.count() - return other -} - // WriteTo writes c to w. func (c *container) WriteTo(w io.Writer) (n int64, err error) { if c.isArray() { @@ -1812,6 +1799,43 @@ type ContainerInfo struct { Pointer unsafe.Pointer // offset within the mmap } +// flip returns a new container containing the inverse of all +// bits in a. +func flip(a *container) *container { + if a.isArray() { + return flipArray(a) + } else if a.isRun() { + return flipRun(a) + } else { + return flipBitmap(a) + } +} + +func flipArray(b *container) *container { + // TODO: actually implement this + x := b.clone() + x.arrayToBitmap() + return flipBitmap(x) +} + +func flipBitmap(b *container) *container { + other := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} + + for i, bitmap := range b.bitmap { + other.bitmap[i] = ^bitmap + } + + other.n = other.count() + return other +} + +func flipRun(b *container) *container { + // TODO: actually implement this + x := b.clone() + x.runToBitmap() + return flipBitmap(x) +} + func intersectionCount(a, b *container) int { if a.isArray() { if b.isArray() { @@ -2571,7 +2595,7 @@ RUNLOOP: func differenceRunBitmap(a, b *container) *container { // If a is full, difference is the flip of b. if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 { - return b.flipBitmap() + return flipBitmap(b) } output := &container{containerType: ContainerRun} output.n = a.n diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index ea6d86cd9..417ac7db6 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -220,8 +220,11 @@ func runEvenBitsSet() []interval16 { /////////////////////////////////////////////////////////////////////////// +// f is a container function taking either one or two containers as input +// func(a *container) *container +// func(a, b *container) *container type testOp struct { - f func(a, b *container) *container + f interface{} x string y string exp string diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index dca413811..216aa7eb7 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -2020,38 +2020,6 @@ func TestXorRunRun(t *testing.T) { } } -func TestBitmapFlip(t *testing.T) { - c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} - - ttable := []struct { - original uint64 - flipped uint64 - }{ - {0x0000000000000000, 0xFFFFFFFFFFFFFFFF}, - {0xFFFFFFFFFFFFFFFF, 0x0000000000000000}, - {0xFFFFFFFFFFFFFFF0, 0x000000000000000F}, - {0xFFFFFFEFFFFFFFFF, 0x0000001000000000}, - {0x0000001000000000, 0xFFFFFFEFFFFFFFFF}, - } - - expectedN := int(65536) - for i, tt := range ttable { - c.bitmap[i] = tt.original - expectedN -= int(popcount(tt.original)) - } - - o := c.flipBitmap() - - for i, tt := range ttable { - if o.bitmap[i] != tt.flipped { - t.Fatalf("bitmapFlip calculation. expected %v, got %v", tt.flipped, o.bitmap[i]) - } - } - if o.n != expectedN { - t.Fatalf("bitmapFlip calculation. expected count %v, got %v", expectedN, o.n) - } -} - func TestBitmapXorRange(t *testing.T) { c := &container{bitmap: make([]uint64, bitmapN), containerType: ContainerBitmap} tests := []struct { @@ -3185,12 +3153,24 @@ func TestContainerCombinations(t *testing.T) { //{xor, "evenBitsSet", "outerBitsSet", ""}, {xor, "evenBitsSet", "oddBitsSet", "full"}, {xor, "evenBitsSet", "evenBitsSet", "empty"}, + + // flip + {flip, "empty", "", "full"}, + {flip, "full", "", "empty"}, + {flip, "firstBitSet", "", "firstBitUnset"}, + {flip, "lastBitSet", "", "lastBitUnset"}, + {flip, "firstBitUnset", "", "firstBitSet"}, + {flip, "lastBitUnset", "", "lastBitSet"}, + {flip, "innerBitsSet", "", "outerBitsSet"}, + {flip, "outerBitsSet", "", "innerBitsSet"}, + {flip, "oddBitsSet", "", "evenBitsSet"}, + {flip, "evenBitsSet", "", "oddBitsSet"}, } for _, testOp := range testOps { for _, x := range containerTypes { for _, y := range containerTypes { desc := fmt.Sprintf("%s(%s/%s, %s/%s)", getFunctionName(testOp.f), cm[x], testOp.x, cm[y], testOp.y) - ret := testOp.f(cts[x][testOp.x], cts[y][testOp.y]) + ret := runContainerFunc(testOp.f, cts[x][testOp.x], cts[y][testOp.y]) exp := testOp.exp // Convert to all container types and check result. @@ -3240,3 +3220,14 @@ func TestContainerCombinations(t *testing.T) { } } } + +//func getFunc(func(a, b *container) *container, m, n *container) *container { +func runContainerFunc(f interface{}, c ...*container) *container { + switch f.(type) { + case func(*container) *container: + return f.(func(*container) *container)(c[0]) + case func(*container, *container) *container: + return f.(func(a, b *container) *container)(c[0], c[1]) + } + return nil +} From 28acc29a109433c69927889624099c2aed740b67 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 10:37:20 -0500 Subject: [PATCH 49/79] refactoring pilosa/server trying to separate internal an external concerns in pilosa.Server - it should handle Cluster, Holder, etc. while pilosa/server handles things with external deps - e.g. Logger, Stats, Handler, etc. Using functional options in pilosa.Server now. --- broadcast_test.go | 11 +- client_test.go | 3 +- cmd/server.go | 54 +--------- cmd/server_test.go | 12 +-- ctl/common.go | 2 +- ctl/server.go | 4 - handler.go | 7 -- holder.go | 1 - holder_test.go | 5 +- server.go | 228 +++++++++++++++++++++++++---------------- server/cluster_test.go | 69 +------------ server/server.go | 215 ++++++++++++++++++++++++-------------- server/server_test.go | 6 +- test/executor.go | 3 +- test/pilosa.go | 23 ++--- test/test.go | 7 -- 16 files changed, 311 insertions(+), 339 deletions(-) diff --git a/broadcast_test.go b/broadcast_test.go index f4ff4c0fa..187e50b4e 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -15,12 +15,16 @@ package pilosa_test import ( + "bytes" "reflect" "testing" + "io/ioutil" + "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" + "github.com/pilosa/pilosa/server" ) // Ensure a message can be marshaled and unmarshaled. @@ -52,11 +56,12 @@ func testMessageMarshal(t *testing.T, m proto.Message) { // Ensure that BroadcastReceiver can register a BroadcastHandler. func TestBroadcast_BroadcastReceiver(t *testing.T) { - - s, err := pilosa.NewServer() + com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard) + err := com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO if err != nil { - t.Fatalf("getting new server: %v", err) + t.Fatalf("setting up server: %v", err) } + s := com.Server sbr := NewSimpleBroadcastReceiver() sbh := NewSimpleBroadcastHandler() diff --git a/client_test.go b/client_test.go index 4bd0bade6..d11ab16a9 100644 --- a/client_test.go +++ b/client_test.go @@ -26,6 +26,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -47,7 +48,7 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { var defaultClient *http.Client func init() { - defaultClient = pilosa.GetHTTPClient(nil) + defaultClient = server.GetHTTPClient(nil) } diff --git a/cmd/server.go b/cmd/server.go index 81633ea01..93ecb543b 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -15,17 +15,11 @@ package cmd import ( - "fmt" "io" - "os" - "os/signal" - "runtime/pprof" - "syscall" - "time" + "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/ctl" "github.com/pilosa/pilosa/server" ) @@ -45,52 +39,10 @@ It will load existing data from the configured directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - // Set up the logger. - if err := Server.SetupLogger(); err != nil { - return fmt.Errorf("error setting up the logger: %v", err) - } - logger := Server.Server.Logger - logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime) - - // Start CPU profiling. - if Server.CPUProfile != "" { - f, err := os.Create(Server.CPUProfile) - if err != nil { - return fmt.Errorf("create cpu profile: %v", err) - } - defer f.Close() - - fmt.Fprintln(Server.Stderr, "Starting cpu profile") - pprof.StartCPUProfile(f) - time.AfterFunc(Server.CPUTime, func() { - fmt.Fprintln(Server.Stderr, "Stopping cpu profile") - pprof.StopCPUProfile() - f.Close() - }) - } - - // Execute the program. if err := Server.Run(); err != nil { - return fmt.Errorf("error running server: %v", err) + return errors.Wrap(err, "running server") } - - // First SIGKILL causes server to shut down gracefully. - c := make(chan os.Signal, 2) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - select { - case sig := <-c: - logger.Printf("Received %s; gracefully shutting down...\n", sig.String()) - - // Second signal causes a hard shutdown. - go func() { <-c; os.Exit(1) }() - - if err := Server.Close(); err != nil { - return err - } - case <-Server.Done: - logger.Printf("Server closed externally") - } - return nil + return errors.Wrap(Server.Wait(), "waiting on Server") }, } diff --git a/cmd/server_test.go b/cmd/server_test.go index e69ce1134..abbe8d7a4 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -37,8 +37,6 @@ func TestServerHelp(t *testing.T) { func TestServerConfig(t *testing.T) { actualDataDir, err := ioutil.TempDir("", "") failErr(t, err, "making data dir") - profFile, err := ioutil.TempFile("", "") - failErr(t, err, "making temp file") logFile, err := ioutil.TempFile("", "") failErr(t, err, "making log file") tests := []commandTest{ @@ -93,7 +91,7 @@ func TestServerConfig(t *testing.T) { // TEST 2 { args: []string{"server", "--log-path", logFile.Name(), "--cluster.disabled", "true"}, - env: map[string]string{"PILOSA_PROFILE_CPU_TIME": "1m"}, + env: map[string]string{}, cfgFileContent: ` bind = "localhost:19444" data-dir = "` + actualDataDir + `" @@ -103,9 +101,6 @@ func TestServerConfig(t *testing.T) { ] [anti-entropy] interval = "11m0s" - [profile] - cpu = "` + profFile.Name() + `" - cpu-time = "35s" [metric] service = "statsd" host = "127.0.0.1:8125" @@ -114,8 +109,6 @@ func TestServerConfig(t *testing.T) { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:19444"}) v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*11)) - v.Check(cmd.Server.CPUProfile, profFile.Name()) - v.Check(cmd.Server.CPUTime, time.Minute) v.Check(cmd.Server.Config.LogPath, logFile.Name()) v.Check(cmd.Server.Config.Metric.Service, "statsd") v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125") @@ -147,6 +140,9 @@ func TestServerConfig(t *testing.T) { case <-cmd.Server.Started: case <-executed: } + if execErr != nil { + t.Fatalf("executing server command: %v", execErr) + } err := cmd.Server.Close() failErr(t, err, "closing pilosa server command") <-executed diff --git a/ctl/common.go b/ctl/common.go index 6ebcd3132..3629e233e 100644 --- a/ctl/common.go +++ b/ctl/common.go @@ -49,7 +49,7 @@ func CommandClient(cmd CommandWithTLSSupport) (*pilosa.InternalHTTPClient, error InsecureSkipVerify: tlsConfig.SkipVerify, } } - client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), pilosa.GetHTTPClient(TLSConfig)) + client, err := pilosa.NewInternalHTTPClient(cmd.TLSHost(), server.GetHTTPClient(TLSConfig)) if err != nil { return nil, err } diff --git a/ctl/server.go b/ctl/server.go index 1c60a7c9c..4816f6170 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -61,8 +61,4 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.") flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.") flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.") - - // CPU Profiling - flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") - flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") } diff --git a/handler.go b/handler.go index bd97c1d1a..52cbff094 100644 --- a/handler.go +++ b/handler.go @@ -15,7 +15,6 @@ package pilosa import ( - "context" "encoding/json" "expvar" "fmt" @@ -35,7 +34,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/gorilla/mux" "github.com/pilosa/pilosa/internal" - "github.com/pilosa/pilosa/pql" "github.com/pkg/errors" ) @@ -45,11 +43,6 @@ type Handler struct { FileSystem FileSystem - // The execution engine for running queries. - Executor interface { - Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) - } - Logger Logger // Keeps the query argument validators for each handler diff --git a/holder.go b/holder.go index fa93767a3..2279a3e19 100644 --- a/holder.go +++ b/holder.go @@ -530,7 +530,6 @@ func (h *Holder) setFileLimit() { func (h *Holder) loadNodeID() (string, error) { idPath := path.Join(h.Path, "ID") nodeID := "" - h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.Path, 0777); err != nil { return "", err diff --git a/holder_test.go b/holder_test.go index 8b1f6166a..7053fa1f2 100644 --- a/holder_test.go +++ b/holder_test.go @@ -25,6 +25,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) @@ -367,7 +368,7 @@ func TestHolder_DeleteIndex(t *testing.T) { // Ensure holder can sync with a remote holder. func TestHolderSyncer_SyncHolder(t *testing.T) { cluster := test.NewCluster(2) - client := pilosa.GetHTTPClient(nil) + client := server.GetHTTPClient(nil) // Create a local holder. hldr0 := test.MustOpenHolder() defer hldr0.Close() @@ -451,7 +452,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { Holder: hldr0.Holder, Node: cluster.Nodes[0], Cluster: cluster, - RemoteClient: pilosa.GetHTTPClient(nil), + RemoteClient: server.GetHTTPClient(nil), Stats: pilosa.NopStatsClient, } diff --git a/server.go b/server.go index 6d25e3320..f022d562a 100644 --- a/server.go +++ b/server.go @@ -55,55 +55,154 @@ func OptServerLogger(l Logger) ServerOption { } } +func OptServerReplicaN(n int) ServerOption { + return func(s *Server) error { + s.Cluster.ReplicaN = n + return nil + } +} + +func OptServerDataDir(dir string) ServerOption { + return func(s *Server) error { + s.Cluster.Path = dir + s.Holder.Path = dir + return nil + } +} + +func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { + return func(s *Server) error { + s.NewAttrStore = af + s.Holder.NewAttrStore = af + return nil + } +} + +func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { + return func(s *Server) error { + s.AntiEntropyInterval = interval + return nil + } +} + +func OptServerLongQueryTime(dur time.Duration) ServerOption { + return func(s *Server) error { + s.Cluster.LongQueryTime = dur + return nil + } +} + +func OptServerHandler(h *Handler) ServerOption { + return func(s *Server) error { + s.Handler = h + return nil + } +} + +func OptServerMaxWritesPerRequest(n int) ServerOption { + return func(s *Server) error { + s.MaxWritesPerRequest = n + return nil + } +} + +func OptServerMetricInterval(dur time.Duration) ServerOption { + return func(s *Server) error { + s.MetricInterval = dur + return nil + } +} + +func OptServerSystemInfo(si SystemInfo) ServerOption { + return func(s *Server) error { + s.SystemInfo = si + return nil + } +} + +func OptServerGCNotifier(gcn GCNotifier) ServerOption { + return func(s *Server) error { + s.GCNotifier = gcn + return nil + } +} + +func OptServerRemoteClient(c *http.Client) ServerOption { + return func(s *Server) error { + s.RemoteClient = c + s.Cluster.RemoteClient = c + return nil + } +} + +func OptServerStatsClient(sc StatsClient) ServerOption { + return func(s *Server) error { + s.Holder.Stats = sc + return nil + } +} + +func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { + return func(s *Server) error { + s.DiagnosticInterval = dur + return nil + } +} + +func OptServerListener(ln net.Listener) ServerOption { + return func(s *Server) error { + s.ln = ln + + return nil + } +} + +func OptServerURI(uri *URI) ServerOption { + return func(s *Server) error { + s.URI = *uri + return nil + } +} + // Server represents a holder wrapped by a running HTTP server. type Server struct { - ln net.Listener - // Close management. wg sync.WaitGroup closing chan struct{} - // Data storage and HTTP interface. - Holder *Holder + // Internal + Holder *Holder + Cluster *Cluster + diagnostics *DiagnosticsCollector + + // External Handler *Handler Broadcaster Broadcaster BroadcastReceiver BroadcastReceiver Gossiper Gossiper RemoteClient *http.Client + SystemInfo SystemInfo + GCNotifier GCNotifier + NewAttrStore func(string) AttrStore + Logger Logger + TLS *tls.Config + ln net.Listener - // Cluster configuration. - Network string - NodeID string - URI URI - Cluster *Cluster - diagnostics *DiagnosticsCollector - SystemInfo SystemInfo - - GCNotifier GCNotifier - - NewAttrStore func(string) AttrStore - - // Background monitoring intervals. + NodeID string + URI URI AntiEntropyInterval time.Duration MetricInterval time.Duration DiagnosticInterval time.Duration - - // TLS configuration - TLS *tls.Config - - // Misc options. MaxWritesPerRequest int - Logger Logger - defaultClient InternalClient } // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ - closing: make(chan struct{}), - + closing: make(chan struct{}), + Cluster: NewCluster(), Holder: NewHolder(), Handler: NewHandler(), Broadcaster: NopBroadcaster, @@ -111,8 +210,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), SystemInfo: NewNopSystemInfo(), - Network: "tcp", - GCNotifier: NopGCNotifier, NewAttrStore: NewNopAttrStore, @@ -131,20 +228,25 @@ func NewServer(opts ...ServerOption) (*Server, error) { } } - s.Handler.API = NewAPI() - s.Handler.API.Holder = s.Holder + s.Holder.Logger = s.Logger + s.Holder.Stats.SetLogger(s.Logger) + + s.Cluster.Logger = s.Logger + s.Cluster.Holder = s.Holder + s.Cluster.RemoteClient = s.RemoteClient + // update URI port with actual listener port. TODO this should probably be done outside of here. + if s.URI.Port() == 0 { + s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) + } return s, nil } // Open opens and initializes the server. func (s *Server) Open() error { - s.Handler.API.Logger = s.Logger // TODO do this in NewServer with functional options s.Logger.Printf("open server") // s.ln can be configured prior to Open() via s.OpenListener(). if s.ln == nil { - if err := s.OpenListener(); err != nil { - return err - } + return errors.New("Must pass a listener option to NewServer") } // Get or create NodeID. @@ -181,13 +283,13 @@ func (s *Server) Open() error { s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest // Initialize HTTP handler. + s.Handler.API.Holder = s.Holder s.Handler.API.Broadcaster = s.Broadcaster s.Handler.API.BroadcastHandler = s s.Handler.API.StatusHandler = s s.Handler.API.URI = s.URI s.Handler.API.Cluster = s.Cluster s.Handler.API.Executor = e - s.Handler.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster @@ -234,43 +336,6 @@ func (s *Server) Open() error { return nil } -// OpenListener opens a listener for the Server. -func (s *Server) OpenListener() error { - s.Logger.Printf("open server listener: %s", s.URI) - if s.ln != nil { - return fmt.Errorf("a listener already exists for server: %s", s.URI) - } - - var ln net.Listener - var err error - - // If bind URI has the https scheme, enable TLS - if s.URI.Scheme() == "https" && s.TLS != nil { - ln, err = tls.Listen("tcp", s.URI.HostPort(), s.TLS) - if err != nil { - return err - } - } else if s.URI.Scheme() == "http" { - // Open HTTP listener to determine port (if specified as :0). - ln, err = net.Listen(s.Network, s.URI.HostPort()) - if err != nil { - return fmt.Errorf("net.Listen: %v", err) - } - } else { - return fmt.Errorf("unsupported scheme: %s", s.URI.Scheme()) - } - - s.ln = ln - - if s.URI.Port() == 0 { - // If the port is 0, it is set automatically. - // Find out automatically set port and update the host. - s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) - } - - return nil -} - // Close closes the server and waits for it to shutdown. func (s *Server) Close() error { // Notify goroutines to stop. @@ -311,25 +376,6 @@ func (s *Server) Addr() net.Addr { } return s.ln.Addr() } -func GetHTTPClient(t *tls.Config) *http.Client { - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - MaxIdleConns: 1000, - MaxIdleConnsPerHost: 200, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - if t != nil { - transport.TLSClientConfig = t - } - return &http.Client{Transport: transport} -} func (s *Server) monitorAntiEntropy() { ticker := time.NewTicker(s.AntiEntropyInterval) diff --git a/server/cluster_test.go b/server/cluster_test.go index 5b0321285..1a64ce305 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -26,81 +26,16 @@ import ( "golang.org/x/sync/errgroup" "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/test" ) // Ensure program can send/receive broadcast messages. func TestMain_SendReceiveMessage(t *testing.T) { - - m0 := test.MustRunMain() + ms := test.MustRunMainWithCluster(t, 2) + m0, m1 := ms[0], ms[1] defer m0.Close() - - m1 := test.MustRunMain() defer m1.Close() - // Update cluster config - m0.Server.Cluster.Nodes = []*pilosa.Node{ - {ID: m0.Server.NodeID, URI: m0.Server.URI}, - {ID: m1.Server.NodeID, URI: m1.Server.URI}, - } - m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes - - // Configure node0 - - // get the host portion of addr to use for binding - m0.Config.Gossip.Port = "0" - m0.Config.Gossip.Seeds = []string{} - - m0.Server.Cluster.Coordinator = m0.Server.NodeID - m0.Server.Cluster.Topology = &pilosa.Topology{NodeIDs: []string{m0.Server.NodeID, m1.Server.NodeID}} - m0.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m0.Server.Logger) - gossipMemberSet0, err := gossip.NewGossipMemberSet(m0.Server.URI.HostPort(), m0.Server.URI.Host(), m0.Config.Gossip, m0.Server) - if err != nil { - t.Fatal(err) - } - m0.Server.Cluster.MemberSet = gossipMemberSet0 - m0.Server.Broadcaster = m0.Server - m0.Server.Gossiper = gossipMemberSet0 - m0.Server.Handler.API.Broadcaster = m0.Server.Broadcaster - m0.Server.Holder.Broadcaster = m0.Server.Broadcaster - m0.Server.BroadcastReceiver = gossipMemberSet0 - - if err := m0.Server.BroadcastReceiver.Start(m0.Server); err != nil { - t.Fatal(err) - } - // Open Cluster management. - if err := m0.Server.Cluster.Open(); err != nil { - t.Fatal(err) - } - - // Configure node1 - - // get the host portion of addr to use for binding - m1.Config.Gossip.Port = "0" - m1.Config.Gossip.Seeds = []string{gossipMemberSet0.GetBindAddr()} - - m1.Server.Cluster.Coordinator = m0.Server.NodeID - m1.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m1.Server.Logger) - gossipMemberSet1, err := gossip.NewGossipMemberSet(m1.Server.URI.HostPort(), m1.Server.URI.Host(), m1.Config.Gossip, m1.Server) - if err != nil { - t.Fatal(err) - } - m1.Server.Cluster.MemberSet = gossipMemberSet1 - m1.Server.Broadcaster = m1.Server - m1.Server.Gossiper = gossipMemberSet1 - m1.Server.Handler.API.Broadcaster = m1.Server.Broadcaster - m1.Server.Holder.Broadcaster = m1.Server.Broadcaster - m1.Server.BroadcastReceiver = gossipMemberSet1 - - if err := m1.Server.BroadcastReceiver.Start(m1.Server); err != nil { - t.Fatal(err) - } - // Open Cluster management. - if err := m1.Server.Cluster.Open(); err != nil { - t.Fatal(err) - } - m0.Server.Cluster.SetState(pilosa.ClusterStateNormal) m1.Server.Cluster.SetState(pilosa.ClusterStateNormal) diff --git a/server/server.go b/server/server.go index 2606ff248..829676fd5 100644 --- a/server/server.go +++ b/server/server.go @@ -24,10 +24,14 @@ import ( "io" "log" "math/rand" + "net" + "net/http" "os" + "os/signal" "path/filepath" "strconv" "strings" + "syscall" "time" "crypto/tls" @@ -46,10 +50,10 @@ func init() { rand.Seed(time.Now().UTC().UnixNano()) } -const ( - // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" -) +type loggerLogger interface { + pilosa.Logger + Logger() *log.Logger +} // Command represents the state of the pilosa server command. type Command struct { @@ -58,10 +62,6 @@ type Command struct { // Configuration. Config *Config - // Profiling options. - CPUProfile string - CPUTime time.Duration - // Gossip transport GossipTransport *gossip.Transport @@ -75,18 +75,12 @@ type Command struct { // Passed to the Gossip implementation. logOutput io.Writer - logger *log.Logger + logger loggerLogger } // NewCommand returns a new instance of Main. func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { - s, err := pilosa.NewServer() - if err != nil { - panic(err) - } - return &Command{ - Server: s, Config: NewConfig(), CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), @@ -97,7 +91,7 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { } // Run executes the pilosa server. -func (m *Command) Run(args ...string) (err error) { +func (m *Command) Run(args ...string) (err error) { // TODO args WTF defer close(m.Started) prefix := "~" + string(filepath.Separator) if strings.HasPrefix(m.Config.DataDir, prefix) { @@ -125,76 +119,72 @@ func (m *Command) Run(args ...string) (err error) { return fmt.Errorf("server.Open: %v", err) } - m.Server.Logger.Printf("Listening as %s\n", m.Server.URI) + m.logger.Printf("Listening as %s\n", m.Server.URI) + return nil } +// Wait waits for the server to be closed or interrupted. +func (m *Command) Wait() error { + // First SIGKILL causes server to shut down gracefully. + c := make(chan os.Signal, 2) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + select { + case sig := <-c: + m.logger.Printf("Received %s; gracefully shutting down...\n", sig.String()) + + // Second signal causes a hard shutdown. + go func() { <-c; os.Exit(1) }() + return errors.Wrap(m.Close(), "closing command") + case <-m.Done: + m.logger.Printf("Server closed externally") + return nil + } +} + // SetupLogger sets up the logger based on the configuration. -func (m *Command) SetupLogger() error { +func (m *Command) SetupLogger() (pilosa.Logger, error) { + if m.logger != nil { + return m.logger, nil + } var err error if m.Config.LogPath == "" { m.logOutput = m.Stderr } else { m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) if err != nil { - return err + return nil, errors.Wrap(err, "opening file") } } if m.Config.Verbose { - vbl := pilosa.NewVerboseLogger(m.logOutput) - m.logger = vbl.Logger() - m.Server.Logger = vbl + m.logger = pilosa.NewVerboseLogger(m.logOutput) } else { - sl := pilosa.NewStandardLogger(m.logOutput) - m.logger = sl.Logger() - m.Server.Logger = sl + m.logger = pilosa.NewStandardLogger(m.logOutput) } - return nil + return m.logger, nil } // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { - m.Server.Handler.Logger = m.Server.Logger - m.Server.Holder.Logger = m.Server.Logger - m.Server.Holder.Stats.SetLogger(m.Server.Logger) + if m.logger == nil { + _, err := m.SetupLogger() + if err != nil { + return errors.Wrap(err, "setting up logger") + } + + } + + handler := pilosa.NewHandler() + handler.Logger = m.logger + handler.FileSystem = &statik.FileSystem{} + handler.API = pilosa.NewAPI() + handler.API.Logger = m.logger uri, err := pilosa.AddressWithDefaults(m.Config.Bind) - if err != nil { - return err + return errors.Wrap(err, "processing bind address") } - m.Server.URI = *uri - - cluster := pilosa.NewCluster() - cluster.ReplicaN = m.Config.Cluster.ReplicaN - cluster.Holder = m.Server.Holder - cluster.Logger = m.Server.Logger - - m.Server.Cluster = cluster - - // Configure data directory (for Cluster .topology) - m.Server.Cluster.Path = m.Config.DataDir - - m.Server.NewAttrStore = boltdb.NewAttrStore - m.Server.Holder.NewAttrStore = boltdb.NewAttrStore - - // Configure holder. - m.Server.Logger.Printf("Using data from: %s\n", m.Config.DataDir) - m.Server.Holder.Path = m.Config.DataDir - m.Server.MetricInterval = time.Duration(m.Config.Metric.PollInterval) - if m.Config.Metric.Diagnostics { - m.Server.DiagnosticInterval = time.Duration(DefaultDiagnosticsInterval) - } - m.Server.SystemInfo = gopsutil.NewSystemInfo() - m.Server.GCNotifier = gcnotify.NewActiveGCNotifier() - m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) - if err != nil { - return err - } - - // Copy configuration flags. - m.Server.MaxWritesPerRequest = m.Config.MaxWritesPerRequest // Setup TLS var TLSConfig *tls.Config @@ -207,27 +197,73 @@ func (m *Command) SetupServer() error { } cert, err := tls.LoadX509KeyPair(m.Config.TLS.CertificatePath, m.Config.TLS.CertificateKeyPath) if err != nil { - return err + return errors.Wrap(err, "load x509 key pair") } - m.Server.TLS = &tls.Config{ + TLSConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: m.Config.TLS.SkipVerify, } - - TLSConfig = m.Server.TLS } - c := pilosa.GetHTTPClient(TLSConfig) - m.Server.RemoteClient = c - m.Server.Handler.API.RemoteClient = c - m.Server.Cluster.RemoteClient = c - // Statik file system. - m.Server.Handler.FileSystem = &statik.FileSystem{} + diagnosticsInterval := time.Duration(0) + if m.Config.Metric.Diagnostics { + diagnosticsInterval = time.Duration(DefaultDiagnosticsInterval) + } - // Set configuration options. - m.Server.AntiEntropyInterval = time.Duration(m.Config.AntiEntropy.Interval) - m.Server.Cluster.LongQueryTime = time.Duration(m.Config.Cluster.LongQueryTime) - return nil + statsClient, err := NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host) + if err != nil { + return errors.Wrap(err, "new stats client") + } + + ln, err := getListener(*uri, TLSConfig) + if err != nil { + return errors.Wrap(err, "getting listener") + } + + c := GetHTTPClient(TLSConfig) + handler.API.RemoteClient = c + + m.Server, err = pilosa.NewServer( + pilosa.OptServerAntiEntropyInterval(time.Duration(m.Config.AntiEntropy.Interval)), + pilosa.OptServerLongQueryTime(time.Duration(m.Config.Cluster.LongQueryTime)), + pilosa.OptServerDataDir(m.Config.DataDir), + pilosa.OptServerReplicaN(m.Config.Cluster.ReplicaN), + pilosa.OptServerMaxWritesPerRequest(m.Config.MaxWritesPerRequest), + pilosa.OptServerMetricInterval(time.Duration(m.Config.Metric.PollInterval)), + pilosa.OptServerDiagnosticsInterval(diagnosticsInterval), + + pilosa.OptServerLogger(m.logger), + pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore), + pilosa.OptServerHandler(handler), + pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()), + pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()), + pilosa.OptServerStatsClient(statsClient), + pilosa.OptServerListener(ln), + pilosa.OptServerURI(uri), + pilosa.OptServerRemoteClient(c), + ) + + return errors.Wrap(err, "new server") +} + +func GetHTTPClient(t *tls.Config) *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 200, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + if t != nil { + transport.TLSClientConfig = t + } + return &http.Client{Transport: transport} } // SetupNetworking sets up internode communication based on the configuration. @@ -266,7 +302,7 @@ func (m *Command) SetupNetworking() error { if m.GossipTransport != nil { transport = m.GossipTransport } else { - transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger) + transport, err = gossip.NewTransport(gossipHost, gossipPort, m.logger.Logger()) if err != nil { return err } @@ -277,8 +313,8 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.Coordinator = m.Server.NodeID } - m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.Server.Logger) - gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, m.Server, gossip.WithLogger(m.logger), gossip.WithTransport(transport)) + m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.logger) + gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport)) if err != nil { return err } @@ -318,3 +354,24 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, none].") } } + +// OpenListener opens a listener for the Server. +func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { + // If bind URI has the https scheme, enable TLS + if uri.Scheme() == "https" && tlsconf != nil { + ln, err = tls.Listen("tcp", uri.HostPort(), tlsconf) + if err != nil { + return nil, errors.Wrap(err, "tls.Listener") + } + } else if uri.Scheme() == "http" { + // Open HTTP listener to determine port (if specified as :0). + ln, err = net.Listen("tcp", uri.HostPort()) + if err != nil { + return nil, errors.Wrap(err, "net.Listen") + } + } else { + return nil, errors.Errorf("unsupported scheme: %s", uri.Scheme()) + } + + return ln, nil +} diff --git a/server/server_test.go b/server/server_test.go index 532257c13..75e0e2242 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -45,7 +45,7 @@ func TestMain_Set_Quick(t *testing.T) { defer m.Close() // Create client. - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { t.Fatal(err) } @@ -322,11 +322,11 @@ func TestMain_FrameRestore(t *testing.T) { defer m21.Close() // Import from first cluster. - client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { t.Fatal("new client:", err) } - client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { t.Fatal("new client:", err) } diff --git a/test/executor.go b/test/executor.go index afe244ad7..8cd6391c3 100644 --- a/test/executor.go +++ b/test/executor.go @@ -20,6 +20,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/server" ) // Executor represents a test wrapper for pilosa.Executor. @@ -30,7 +31,7 @@ type Executor struct { var remoteClient *http.Client func init() { - remoteClient = pilosa.GetHTTPClient(nil) + remoteClient = server.GetHTTPClient(nil) } // NewExecutor returns a new instance of Executor. diff --git a/test/pilosa.go b/test/pilosa.go index b00c9841c..366facdd4 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -49,15 +49,13 @@ func NewMain() *Main { } m := &Main{Command: server.NewCommand(os.Stdin, os.Stdout, os.Stderr)} - m.Server.Network = *Network - m.Server.NewAttrStore = NewAttrStore - m.Server.Holder.NewAttrStore = NewAttrStore m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true m.Command.Stdin = &m.Stdin m.Command.Stdout = &m.Stdout m.Command.Stderr = &m.Stderr + m.SetupServer() if testing.Verbose() { m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) @@ -101,6 +99,7 @@ func runMainWithCluster(size int) ([]*Main, error) { for i := 0; i < size; i++ { m := NewMainWithCluster(i == 0) + m.Config.Cluster.Disabled = false gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i]) if err != nil { @@ -136,12 +135,16 @@ func (m *Main) Reopen() error { } // Create new main with the same config. - config := m.Config + config := m.Command.Config m.Command = server.NewCommand(os.Stdin, os.Stdout, os.Stderr) - m.Server.Network = *Network + m.Command.Config = config + err := m.SetupServer() + if err != nil { + return errors.Wrap(err, "setting up server") + } + m.Server.NewAttrStore = boltdb.NewAttrStore m.Server.Holder.NewAttrStore = m.Server.NewAttrStore - m.Config = config // Run new program. if err := m.Run(); err != nil { @@ -174,12 +177,6 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) ( return seed, err } - // Open server listener. - err = m.Server.OpenListener() - if err != nil { - return seed, err - } - // Open gossip transport to use in SetupServer. transport, err := gossip.NewTransport(host, bindPort, nil) if err != nil { @@ -221,7 +218,7 @@ func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } // Client returns a client to connect to the program. func (m *Main) Client() *pilosa.InternalHTTPClient { - client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), pilosa.GetHTTPClient(nil)) + client, err := pilosa.NewInternalHTTPClient(m.Server.URI.HostPort(), server.GetHTTPClient(nil)) if err != nil { panic(err) } diff --git a/test/test.go b/test/test.go index c83b657c0..01f980ad2 100644 --- a/test/test.go +++ b/test/test.go @@ -13,10 +13,3 @@ // limitations under the License. package test - -import "flag" - -// Test flags. -var ( - Network = flag.String("network", "tcp", "network name") -) From 9f1720f01d3e73989f7dde42b9c187bf25486b36 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 15:12:23 -0500 Subject: [PATCH 50/79] unexport stuff in pilosa.Server refactor gossip.NewGossipMemberset to not take Server --- diagnostics.go | 16 ++--- gossip/gossip.go | 15 ++-- server.go | 182 +++++++++++++++++++++++------------------------ server/server.go | 5 +- 4 files changed, 107 insertions(+), 111 deletions(-) diff --git a/diagnostics.go b/diagnostics.go index 57423d0a5..bfebbb495 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -168,23 +168,23 @@ func (d *DiagnosticsCollector) logErr(err error) bool { // EnrichWithOSInfo adds OS information to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithOSInfo() { - uptime, err := d.server.SystemInfo.Uptime() + uptime, err := d.server.systemInfo.Uptime() if !d.logErr(err) { d.Set("HostUptime", uptime) } - platform, err := d.server.SystemInfo.Platform() + platform, err := d.server.systemInfo.Platform() if !d.logErr(err) { d.Set("OSPlatform", platform) } - family, err := d.server.SystemInfo.Family() + family, err := d.server.systemInfo.Family() if !d.logErr(err) { d.Set("OSFamily", family) } - version, err := d.server.SystemInfo.OSVersion() + version, err := d.server.systemInfo.OSVersion() if !d.logErr(err) { d.Set("OSVersion", version) } - kernelVersion, err := d.server.SystemInfo.KernelVersion() + kernelVersion, err := d.server.systemInfo.KernelVersion() if !d.logErr(err) { d.Set("OSKernelVersion", kernelVersion) } @@ -192,15 +192,15 @@ func (d *DiagnosticsCollector) EnrichWithOSInfo() { // EnrichWithMemoryInfo adds memory information to the diagnostics payload. func (d *DiagnosticsCollector) EnrichWithMemoryInfo() { - memFree, err := d.server.SystemInfo.MemFree() + memFree, err := d.server.systemInfo.MemFree() if !d.logErr(err) { d.Set("MemFree", memFree) } - memTotal, err := d.server.SystemInfo.MemTotal() + memTotal, err := d.server.systemInfo.MemTotal() if !d.logErr(err) { d.Set("MemTotal", memTotal) } - memUsed, err := d.server.SystemInfo.MemUsed() + memUsed, err := d.server.systemInfo.MemUsed() if !d.logErr(err) { d.Set("MemUsed", memUsed) } diff --git a/gossip/gossip.go b/gossip/gossip.go index 8ad00a8a7..e4856ec7b 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -152,7 +152,7 @@ type gossipConfig struct { type GossipMemberSetOption func(*GossipMemberSet) error // WithTransport is a functional option for providing a transport to NewGossipMemberSet. -func WithTransport(transport *Transport) func(*GossipMemberSet) error { +func WithTransport(transport *Transport) GossipMemberSetOption { return func(g *GossipMemberSet) error { g.transport = transport return nil @@ -160,7 +160,7 @@ func WithTransport(transport *Transport) func(*GossipMemberSet) error { } // WithLogger is a functional option for providing a logger to NewGossipMemberSet. -func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { +func WithLogger(logger *log.Logger) GossipMemberSetOption { return func(g *GossipMemberSet) error { g.logger = logger return nil @@ -168,11 +168,8 @@ func WithLogger(logger *log.Logger) func(*GossipMemberSet) error { } // NewGossipMemberSet returns a new instance of GossipMemberSet based on options. -func NewGossipMemberSet(name string, host string, cfg Config, server *pilosa.Server, options ...GossipMemberSetOption) (*GossipMemberSet, error) { - - g := &GossipMemberSet{ - Logger: server.Logger, - } +func NewGossipMemberSet(name string, host string, cfg Config, ger *GossipEventReceiver, sh pilosa.StatusHandler, options ...GossipMemberSetOption) (*GossipMemberSet, error) { + g := &GossipMemberSet{} // options for _, opt := range options { @@ -227,7 +224,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, server *pilosa.Ser // conf.Delegate = g conf.SecretKey = gossipKey - conf.Events = server.Cluster.EventReceiver.(memberlist.EventDelegate) + conf.Events = ger conf.Logger = g.logger g.config = &gossipConfig{ @@ -235,7 +232,7 @@ func NewGossipMemberSet(name string, host string, cfg Config, server *pilosa.Ser gossipSeeds: cfg.Seeds, } - g.statusHandler = server + g.statusHandler = sh return g, nil } diff --git a/server.go b/server.go index f022d562a..d302a7d97 100644 --- a/server.go +++ b/server.go @@ -16,7 +16,6 @@ package pilosa import ( "context" - "crypto/tls" "fmt" "net" "net/http" @@ -45,12 +44,45 @@ var _ Broadcaster = &Server{} var _ BroadcastHandler = &Server{} var _ StatusHandler = &Server{} +// Server represents a holder wrapped by a running HTTP server. +type Server struct { + // Close management. + wg sync.WaitGroup + closing chan struct{} + + // Internal + Holder *Holder + Cluster *Cluster + diagnostics *DiagnosticsCollector + + // External + handler *Handler + Broadcaster Broadcaster + BroadcastReceiver BroadcastReceiver + Gossiper Gossiper + remoteClient *http.Client + systemInfo SystemInfo + gcNotifier GCNotifier + NewAttrStore func(string) AttrStore + logger Logger + ln net.Listener + + NodeID string + URI URI + antiEntropyInterval time.Duration + metricInterval time.Duration + diagnosticInterval time.Duration + maxWritesPerRequest int + + defaultClient InternalClient +} + // ServerOption is a functional option type for pilosa.Server type ServerOption func(s *Server) error func OptServerLogger(l Logger) ServerOption { return func(s *Server) error { - s.Logger = l + s.logger = l return nil } } @@ -80,7 +112,7 @@ func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption { func OptServerAntiEntropyInterval(interval time.Duration) ServerOption { return func(s *Server) error { - s.AntiEntropyInterval = interval + s.antiEntropyInterval = interval return nil } } @@ -94,42 +126,42 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption { func OptServerHandler(h *Handler) ServerOption { return func(s *Server) error { - s.Handler = h + s.handler = h return nil } } func OptServerMaxWritesPerRequest(n int) ServerOption { return func(s *Server) error { - s.MaxWritesPerRequest = n + s.maxWritesPerRequest = n return nil } } func OptServerMetricInterval(dur time.Duration) ServerOption { return func(s *Server) error { - s.MetricInterval = dur + s.metricInterval = dur return nil } } func OptServerSystemInfo(si SystemInfo) ServerOption { return func(s *Server) error { - s.SystemInfo = si + s.systemInfo = si return nil } } func OptServerGCNotifier(gcn GCNotifier) ServerOption { return func(s *Server) error { - s.GCNotifier = gcn + s.gcNotifier = gcn return nil } } func OptServerRemoteClient(c *http.Client) ServerOption { return func(s *Server) error { - s.RemoteClient = c + s.remoteClient = c s.Cluster.RemoteClient = c return nil } @@ -144,7 +176,7 @@ func OptServerStatsClient(sc StatsClient) ServerOption { func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { return func(s *Server) error { - s.DiagnosticInterval = dur + s.diagnosticInterval = dur return nil } } @@ -164,61 +196,27 @@ func OptServerURI(uri *URI) ServerOption { } } -// Server represents a holder wrapped by a running HTTP server. -type Server struct { - // Close management. - wg sync.WaitGroup - closing chan struct{} - - // Internal - Holder *Holder - Cluster *Cluster - diagnostics *DiagnosticsCollector - - // External - Handler *Handler - Broadcaster Broadcaster - BroadcastReceiver BroadcastReceiver - Gossiper Gossiper - RemoteClient *http.Client - SystemInfo SystemInfo - GCNotifier GCNotifier - NewAttrStore func(string) AttrStore - Logger Logger - TLS *tls.Config - ln net.Listener - - NodeID string - URI URI - AntiEntropyInterval time.Duration - MetricInterval time.Duration - DiagnosticInterval time.Duration - MaxWritesPerRequest int - - defaultClient InternalClient -} - // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ closing: make(chan struct{}), Cluster: NewCluster(), Holder: NewHolder(), - Handler: NewHandler(), + handler: NewHandler(), Broadcaster: NopBroadcaster, BroadcastReceiver: NopBroadcastReceiver, diagnostics: NewDiagnosticsCollector(DefaultDiagnosticServer), - SystemInfo: NewNopSystemInfo(), + systemInfo: NewNopSystemInfo(), - GCNotifier: NopGCNotifier, + gcNotifier: NopGCNotifier, NewAttrStore: NewNopAttrStore, - AntiEntropyInterval: time.Minute * 10, - MetricInterval: 0, - DiagnosticInterval: 0, + antiEntropyInterval: time.Minute * 10, + metricInterval: 0, + diagnosticInterval: 0, - Logger: NopLogger, + logger: NopLogger, } for _, opt := range opts { @@ -228,12 +226,12 @@ func NewServer(opts ...ServerOption) (*Server, error) { } } - s.Holder.Logger = s.Logger - s.Holder.Stats.SetLogger(s.Logger) + s.Holder.Logger = s.logger + s.Holder.Stats.SetLogger(s.logger) - s.Cluster.Logger = s.Logger + s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder - s.Cluster.RemoteClient = s.RemoteClient + s.Cluster.RemoteClient = s.remoteClient // update URI port with actual listener port. TODO this should probably be done outside of here. if s.URI.Port() == 0 { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) @@ -243,7 +241,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { // Open opens and initializes the server. func (s *Server) Open() error { - s.Logger.Printf("open server") + s.logger.Printf("open server") // s.ln can be configured prior to Open() via s.OpenListener(). if s.ln == nil { return errors.New("Must pass a listener option to NewServer") @@ -269,36 +267,36 @@ func (s *Server) Open() error { s.Holder.Peek() // Create default HTTP client - s.createDefaultClient(s.RemoteClient) + s.createDefaultClient(s.remoteClient) // Create executor for executing queries. - e := NewExecutor(s.RemoteClient) + e := NewExecutor(s.remoteClient) e.Holder = s.Holder e.Node = node e.Cluster = s.Cluster - e.MaxWritesPerRequest = s.MaxWritesPerRequest + e.MaxWritesPerRequest = s.maxWritesPerRequest // Cluster settings. s.Cluster.Broadcaster = s.Broadcaster - s.Cluster.MaxWritesPerRequest = s.MaxWritesPerRequest + s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest // Initialize HTTP handler. - s.Handler.API.Holder = s.Holder - s.Handler.API.Broadcaster = s.Broadcaster - s.Handler.API.BroadcastHandler = s - s.Handler.API.StatusHandler = s - s.Handler.API.URI = s.URI - s.Handler.API.Cluster = s.Cluster - s.Handler.API.Executor = e + s.handler.API.Holder = s.Holder + s.handler.API.Broadcaster = s.Broadcaster + s.handler.API.BroadcastHandler = s + s.handler.API.StatusHandler = s + s.handler.API.URI = s.URI + s.handler.API.Cluster = s.Cluster + s.handler.API.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster // Serve HTTP. go func() { - err := http.Serve(s.ln, s.Handler) + err := http.Serve(s.ln, s.handler) if err != nil { - s.Logger.Printf("HTTP handler terminated with error: %s\n", err) + s.logger.Printf("HTTP handler terminated with error: %s\n", err) } }() @@ -363,7 +361,7 @@ func (s *Server) LoadNodeID() string { } nodeID, err := s.Holder.loadNodeID() if err != nil { - s.Logger.Printf("loading NodeID: %v", err) + s.logger.Printf("loading NodeID: %v", err) return s.NodeID } return nodeID @@ -378,10 +376,10 @@ func (s *Server) Addr() net.Addr { } func (s *Server) monitorAntiEntropy() { - ticker := time.NewTicker(s.AntiEntropyInterval) + ticker := time.NewTicker(s.antiEntropyInterval) defer ticker.Stop() - s.Logger.Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval) + s.logger.Printf("holder sync monitor initializing (%s interval)", s.antiEntropyInterval) for { // Wait for tick or a close. @@ -392,7 +390,7 @@ func (s *Server) monitorAntiEntropy() { s.Holder.Stats.Count("AntiEntropy", 1, 1.0) } t := time.Now() - s.Logger.Printf("holder sync beginning") + s.logger.Printf("holder sync beginning") // Initialize syncer with local holder and remote client. var syncer HolderSyncer @@ -400,17 +398,17 @@ func (s *Server) monitorAntiEntropy() { syncer.Node = s.Cluster.Node syncer.Cluster = s.Cluster syncer.Closing = s.closing - syncer.RemoteClient = s.RemoteClient + syncer.RemoteClient = s.remoteClient syncer.Stats = s.Holder.Stats.WithTags("HolderSyncer") // Sync holders. if err := syncer.SyncHolder(); err != nil { - s.Logger.Printf("holder sync error: err=%s", err) + s.logger.Printf("holder sync error: err=%s", err) continue } // Record successful sync in log. - s.Logger.Printf("holder sync complete") + s.logger.Printf("holder sync complete") dif := time.Since(t) s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0) } @@ -532,7 +530,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { func (s *Server) SendSync(pb proto.Message) error { var eg errgroup.Group for _, node := range s.Cluster.Nodes { - s.Logger.Printf("SendSync to: %s", node.URI) + s.logger.Printf("SendSync to: %s", node.URI) // Don't forward the message to ourselves. if s.URI == node.URI { continue @@ -554,7 +552,7 @@ func (s *Server) SendAsync(pb proto.Message) error { // SendTo represents an implementation of Broadcaster. func (s *Server) SendTo(to *Node, pb proto.Message) error { - s.Logger.Printf("SendTo: %s", to.URI) + s.logger.Printf("SendTo: %s", to.URI) ctx := context.WithValue(context.Background(), "uri", &to.URI) return s.defaultClient.SendMessage(ctx, pb) } @@ -604,7 +602,7 @@ func (s *Server) HandleRemoteStatus(pb proto.Message) error { err := s.mergeRemoteStatus(pb.(*internal.NodeStatus)) if err != nil { - s.Logger.Printf("merge remote status: %s", err) + s.logger.Printf("merge remote status: %s", err) } }() @@ -629,7 +627,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation if localIndex == nil { - s.Logger.Printf("Local Index not found: %s", index) + s.logger.Printf("Local Index not found: %s", index) continue } if newMax > oldmaxslices[index] { @@ -645,7 +643,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // if we don't know about an index locally, log an error because // indexes should be created and synced prior to slice creation if localIndex == nil { - s.Logger.Printf("Local Index not found: %s", index) + s.logger.Printf("Local Index not found: %s", index) continue } if newMaxInverse > oldMaxInverseSlices[index] { @@ -660,14 +658,14 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { // monitorDiagnostics periodically polls the Pilosa Indexes for cluster info. func (s *Server) monitorDiagnostics() { // Do not send more than once a minute - if s.DiagnosticInterval < time.Minute { - s.Logger.Printf("diagnostics disabled") + if s.diagnosticInterval < time.Minute { + s.logger.Printf("diagnostics disabled") return } else { - s.Logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.DiagnosticInterval) + s.logger.Printf("Pilosa is currently configured to send small diagnostics reports to our team every %v. More information here: https://www.pilosa.com/docs/latest/administration/#diagnostics", s.diagnosticInterval) } - s.diagnostics.Logger = s.Logger + s.diagnostics.Logger = s.logger s.diagnostics.SetVersion(Version) s.diagnostics.Set("Host", s.URI.host) s.diagnostics.Set("Cluster", strings.Join(s.Cluster.NodeIDs(), ",")) @@ -689,11 +687,11 @@ func (s *Server) monitorDiagnostics() { s.diagnostics.CheckVersion() err = s.diagnostics.Flush() if err != nil { - s.Logger.Printf("Diagnostics error: %s", err) + s.logger.Printf("Diagnostics error: %s", err) } } - ticker := time.NewTicker(s.DiagnosticInterval) + ticker := time.NewTicker(s.diagnosticInterval) defer ticker.Stop() flush() for { @@ -710,24 +708,24 @@ func (s *Server) monitorDiagnostics() { // monitorRuntime periodically polls the Go runtime metrics. func (s *Server) monitorRuntime() { // Disable metrics when poll interval is zero. - if s.MetricInterval <= 0 { + if s.metricInterval <= 0 { return } var m runtime.MemStats - ticker := time.NewTicker(s.MetricInterval) + ticker := time.NewTicker(s.metricInterval) defer ticker.Stop() - defer s.GCNotifier.Close() + defer s.gcNotifier.Close() - s.Logger.Printf("runtime stats initializing (%s interval)", s.MetricInterval) + s.logger.Printf("runtime stats initializing (%s interval)", s.metricInterval) for { // Wait for tick or a close. select { case <-s.closing: return - case <-s.GCNotifier.AfterGC(): + case <-s.gcNotifier.AfterGC(): // GC just ran. s.Holder.Stats.Count("garbage_collection", 1, 1.0) case <-ticker.C: diff --git a/server/server.go b/server/server.go index 829676fd5..3e5bb1899 100644 --- a/server/server.go +++ b/server/server.go @@ -313,8 +313,9 @@ func (m *Command) SetupNetworking() error { m.Server.Cluster.Coordinator = m.Server.NodeID } - m.Server.Cluster.EventReceiver = gossip.NewGossipEventReceiver(m.logger) - gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport)) + gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger) + m.Server.Cluster.EventReceiver = gossipEventReceiver + gossipMemberSet, err := gossip.NewGossipMemberSet(m.Server.NodeID, m.Server.URI.Host(), m.Config.Gossip, gossipEventReceiver, m.Server, gossip.WithLogger(m.logger.Logger()), gossip.WithTransport(transport)) if err != nil { return err } From 1e05a6d62725b0e261f09e32ddd4322b6a6ace34 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 15:15:02 -0500 Subject: [PATCH 51/79] fix getListener comment --- server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 3e5bb1899..1c3ba577c 100644 --- a/server/server.go +++ b/server/server.go @@ -356,7 +356,7 @@ func NewStatsClient(name string, host string) (pilosa.StatsClient, error) { } } -// OpenListener opens a listener for the Server. +// getListener gets a net.Listener based on the config. func getListener(uri pilosa.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS if uri.Scheme() == "https" && tlsconf != nil { From 9deefbaefc00366008b2ef25aa7cb880b3497da3 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 16:25:03 -0500 Subject: [PATCH 52/79] unexport setupLogger and simplify --- server/server.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/server/server.go b/server/server.go index 1c3ba577c..6ff80fe7f 100644 --- a/server/server.go +++ b/server/server.go @@ -142,18 +142,15 @@ func (m *Command) Wait() error { } } -// SetupLogger sets up the logger based on the configuration. -func (m *Command) SetupLogger() (pilosa.Logger, error) { - if m.logger != nil { - return m.logger, nil - } +// setupLogger sets up the logger based on the configuration. +func (m *Command) setupLogger() error { var err error if m.Config.LogPath == "" { m.logOutput = m.Stderr } else { m.logOutput, err = os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) if err != nil { - return nil, errors.Wrap(err, "opening file") + return errors.Wrap(err, "opening file") } } @@ -162,17 +159,14 @@ func (m *Command) SetupLogger() (pilosa.Logger, error) { } else { m.logger = pilosa.NewStandardLogger(m.logOutput) } - return m.logger, nil + return nil } // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { - if m.logger == nil { - _, err := m.SetupLogger() - if err != nil { - return errors.Wrap(err, "setting up logger") - } - + err := m.setupLogger() + if err != nil { + return errors.Wrap(err, "setting up logger") } handler := pilosa.NewHandler() From f99479932db488f42c3f61efc37eff5d7c0608fd Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 23 Apr 2018 16:28:24 -0500 Subject: [PATCH 53/79] rename server Run to Start to better reflect functionality --- cmd/server.go | 2 +- server/server.go | 4 ++-- test/pilosa.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 93ecb543b..d906cf189 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -39,7 +39,7 @@ It will load existing data from the configured directory and start listening for client connections on the configured port.`, RunE: func(cmd *cobra.Command, args []string) error { - if err := Server.Run(); err != nil { + if err := Server.Start(); err != nil { return errors.Wrap(err, "running server") } return errors.Wrap(Server.Wait(), "waiting on Server") diff --git a/server/server.go b/server/server.go index 6ff80fe7f..7f5782211 100644 --- a/server/server.go +++ b/server/server.go @@ -90,8 +90,8 @@ func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command { } } -// Run executes the pilosa server. -func (m *Command) Run(args ...string) (err error) { // TODO args WTF +// Start starts the pilosa server - it returns once the server is running. +func (m *Command) Start() (err error) { defer close(m.Started) prefix := "~" + string(filepath.Separator) if strings.HasPrefix(m.Config.DataDir, prefix) { diff --git a/test/pilosa.go b/test/pilosa.go index 366facdd4..a390cd462 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -116,7 +116,7 @@ func runMainWithCluster(size int) ([]*Main, error) { func MustRunMain() *Main { m := NewMain() m.Config.Metric.Diagnostics = false // Disable diagnostics. - if err := m.Run(); err != nil { + if err := m.Start(); err != nil { panic(err) } return m @@ -147,7 +147,7 @@ func (m *Main) Reopen() error { m.Server.Holder.NewAttrStore = m.Server.NewAttrStore // Run new program. - if err := m.Run(); err != nil { + if err := m.Start(); err != nil { return err } return nil From e4a7e2edbd1e22b050f8a584e8936b8044cbfb93 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 16:47:22 -0500 Subject: [PATCH 54/79] remove references to row and column labels from the docs --- docs/api-reference.md | 4 +-- docs/data-model.md | 18 ++++++------- docs/pdk.md | 2 +- docs/query-language.md | 58 +++++++++++++++++++++--------------------- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index a2697f311..9fc232535 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -68,7 +68,7 @@ Sends a query to the Pilosa server with the given index. The request body is UTF ``` request curl localhost:10101/index/user/query \ -X POST \ - -d 'Bitmap(frame="language", rowID=5)' + -d 'Bitmap(frame="language", row=5)' ``` ``` response {"results":[{"attrs":{},"bits":[100]}]} @@ -83,7 +83,7 @@ The query is executed for all [slices](../data-model/#slice) by default. To use ``` request curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \ -X POST \ - -d 'Bitmap(frame="language", rowID=5)' + -d 'Bitmap(frame="language", row=5)' ``` ``` response { diff --git a/docs/data-model.md b/docs/data-model.md index dc8646a85..693b4c97d 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -88,8 +88,8 @@ The standard View contains the same Row/Column format as the input data. If a Frame has a time quantum, then Views are generated for each of the defined time segments. For example, for a frame with a time quantum of `YMD`, the following `SetBit()` queries will result in the data described in the illustration below: ``` -SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-18T00:00") -SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") +SetBit(frame="A", row=8, col=3, timestamp="2017-05-18T00:00") +SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00") ``` ![time quantum frame diagram](/img/docs/frame-time-quantum.svg) @@ -100,17 +100,17 @@ SetBit(frame="A", rowID=8, columnID=3, timestamp="2017-05-19T00:00") Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers. -Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rowIDs' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. +Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rows' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. For example, the following `SetFieldValue()` queries will result in the data described in the illustration below: ``` -SetFieldValue(columnID=1, frame="A", field0=1) -SetFieldValue(columnID=2, frame="A", field0=2) -SetFieldValue(columnID=3, frame="A", field0=3) -SetFieldValue(columnID=4, frame="A", field0=7) -SetFieldValue(columnID=2, frame="A", field1=1) -SetFieldValue(columnID=3, frame="A", field1=6) +SetFieldValue(col=1, frame="A", field0=1) +SetFieldValue(col=2, frame="A", field0=2) +SetFieldValue(col=3, frame="A", field0=3) +SetFieldValue(col=4, frame="A", field0=7) +SetFieldValue(col=2, frame="A", field1=1) +SetFieldValue(col=3, frame="A", field1=6) ``` ![BSI frame diagram](/img/docs/frame-bsi.svg) diff --git a/docs/pdk.md b/docs/pdk.md index 8c02dc279..9be1d70be 100644 --- a/docs/pdk.md +++ b/docs/pdk.md @@ -34,7 +34,7 @@ With this definition available, the PDK tool can run the import, which consists - for each CSV record: - generate a columnID - apply all ParserMappers, generating a list of (frame, ID) pairs - - set the appropriate bit. schematically: SetBit(id=rowID, frame=frame, profileID=columnID) + - set the appropriate bit. schematically: SetBit(row=rowID, frame=frame, col=columnID) The process is summarized in this flowchart: diff --git a/docs/query-language.md b/docs/query-language.md index f26cf021d..83a10ab36 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -25,17 +25,17 @@ There will be one item in the `results` array for each PQL query in the request. * Angle Brackets `<>` denote required arguments * Square Brackets `[]` denote optional arguments -* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ROW_LABEL`, `STRING`) +* UPPER_CASE denotes a descriptor that will need to be filled in with a concrete value (e.g. `ATTR_NAME`, `STRING`) ##### Examples Before running any of the example queries below, follow the instructions in the [Getting Started](../getting-started/) section to set up an index, frames, and populate them with some data. -The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", columnID=10, rowID=1)` against a server using curl, you would: +The examples just show the PQL quer(ies) needed - to run the query `SetBit(frame="stargazer", col=10, row=1)` against a server using curl, you would: ``` request curl localhost:10101/index/repository/query \ -X POST \ - -d 'SetBit(frame="stargazer", columnID=10, rowID=1)' + -d 'SetBit(frame="stargazer", col=10, row=1)' ``` ``` response {"results":[true]} @@ -58,7 +58,7 @@ curl localhost:10101/index/repository/query \ **Spec:** ``` -SetBit(, , , +SetBit(, , , [timestamp=TIMESTAMP]) ``` @@ -76,26 +76,26 @@ A return value of `false` indicates that the bit was already set to 1 and nothin **Examples:** ``` -SetBit(frame="stargazer", columnID=10, rowID=1) +SetBit(frame="stargazer", col=10, row=1) ``` This query illustrates setting a bit in the stargazer frame. User with id=1 has starred repository with id=10. SetBit also supports providing a timestamp. To write the date that a user starred a repository. ``` -SetBit(frame="stargazer", columnID=10, rowID=1, timestamp="2016-01-01T00:00") +SetBit(frame="stargazer", col=10, row=1, timestamp="2016-01-01T00:00") ``` Setting multiple bits in a single request: ``` -SetBit(frame="stargazer", columnID=10, rowID=1) SetBit(frame="stargazer", columnID=10, rowID=2) SetBit(frame="stargazer", columnID=20, rowID=1) SetBit(frame="stargazer", columnID=30, rowID=2) +SetBit(frame="stargazer", col=10, row=1) SetBit(frame="stargazer", col=10, row=2) SetBit(frame="stargazer", col=20, row=1) SetBit(frame="stargazer", col=30, row=2) ``` #### SetRowAttrs **Spec:** ``` -SetRowAttrs(, , +SetRowAttrs(, , , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -111,13 +111,13 @@ SetRowAttrs queries always return `null` upon success. **Examples:** ``` -SetRowAttrs(frame="stargazer", rowID=10, username="mrpi", active=true) +SetRowAttrs(frame="stargazer", row=10, username="mrpi", active=true) ``` Set username value and active status for user 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a row with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", stargazer_id=10)`. ``` -SetRowAttrs(frame="stargazer", rowID=10, username=null) +SetRowAttrs(frame="stargazer", row=10, username=null) ``` Delete username value for user 10. @@ -127,7 +127,7 @@ Delete username value for user 10. **Spec:** ``` -SetColumnAttrs(, , +SetColumnAttrs(, , , [ATTR_NAME=ATTR_VALUE ...]) ``` @@ -143,13 +143,13 @@ SetColumnAttrs queries always return `null` upon success. Setting a value of `nu **Examples:** ``` -SetColumnAttrs(columnID=10, stars=123, url="http://projects.pilosa.com/10", active=true) +SetColumnAttrs(col=10, stars=123, url="http://projects.pilosa.com/10", active=true) ``` -Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", columnID=10)`. +Set url value and active status for project 10. These are arbitrary key/value pairs which have no meaning to Pilosa. You can see the attributes you've set on a column with a [Bitmap](../query-language/#bitmap) query like so `Bitmap(frame="stargazer", col=10)`. ``` -SetColumnAttrs(columnID=10, url=null) +SetColumnAttrs(col=10, url=null) ``` Delete url value for repo 10. @@ -160,7 +160,7 @@ Delete url value for repo 10. **Spec:** ``` -SetBit(, , , +SetBit(, , , [timestamp=TIMESTAMP]) ``` @@ -177,7 +177,7 @@ A return value of `false` indicates that the bit was already set to 0 and nothin **Examples:** ``` -ClearBit(frame="stargazer", columnID=10, rowID=1) +ClearBit(frame="stargazer", col=10, row=1) ``` Remove relationship between the stargazer in row 1 and the repository in column 10 from the stargazer frame. @@ -188,12 +188,12 @@ Remove relationship between the stargazer in row 1 and the repository in column **Spec:** ``` -SetFieldValue(, , ) +SetFieldValue(, , ) ``` **Description:** -`SetFieldValue` assigns an integer value with the specified field name to the `columnID` in the given `frame`. +`SetFieldValue` assigns an integer value with the specified field name to the `col` in the given `frame`. **Result Type:** null @@ -203,7 +203,7 @@ SetFieldValue returns `null` upon success. Set the number of pull requests of repository 10. ``` -SetFieldValue(columnID=10, frame="stats", pullrequests=2) +SetFieldValue(col=10, frame="stats", pullrequests=2) ``` @@ -214,7 +214,7 @@ SetFieldValue(columnID=10, frame="stats", pullrequests=2) **Spec:** ``` -Bitmap(, ( | =UINT)) +Bitmap(, ( | =UINT)) ``` **Description:** @@ -229,7 +229,7 @@ e.g. `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` Query all repositories that user 1 has starred. ``` -Bitmap(frame="stargazer", rowID=1) +Bitmap(frame="stargazer", row=1) ``` Returns `{"attrs":{"username":"mrpi","active":true},"bits":[10, 20]}` @@ -286,7 +286,7 @@ attrs will always be empty Query repositories which have been starred by two users. ``` -Intersect(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2)) +Intersect(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) ``` Returns `{"attrs":{},"bits":[10]}`. @@ -313,7 +313,7 @@ attrs will always be empty Query repositories which have been starred by one user and not another. ``` -Difference(Bitmap(frame="stargazer", rowID=1), Bitmap( frame="stargazer", rowID=2)) +Difference(Bitmap(frame="stargazer", row=1), Bitmap( frame="stargazer", row=2)) ``` Return `{"results":[{"attrs":{},"bits":[20]}]}` @@ -321,7 +321,7 @@ Return `{"results":[{"attrs":{},"bits":[20]}]}` * bits are repositories that were starred by user 1 BUT NOT user 2 ``` -Difference(Bitmap(frame="stargazer", rowID=2), Bitmap( frame="stargazer", rowID=1)) +Difference(Bitmap(frame="stargazer", row=2), Bitmap( frame="stargazer", row=1)) ``` Return `{"attrs":{},"bits":[30]}` @@ -349,7 +349,7 @@ attrs will always be empty Query repositories which have been starred by two users. ``` -Xor(Bitmap(frame="stargazer", rowID=1), Bitmap(frame="stargazer", rowID=2)) +Xor(Bitmap(frame="stargazer", row=1), Bitmap(frame="stargazer", row=2)) ``` Returns `{"attrs":{},"bits":[30]}`. @@ -373,7 +373,7 @@ Returns the number of set bits in the `BITMAP_CALL` passed in. Query the number of repositories to which a user has contributed. ``` -Count(Bitmap(frame="stargazer", rowID=1)) +Count(Bitmap(frame="stargazer", row=1)) ``` Return `2` @@ -427,7 +427,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 2}]` * Results are the top two users sorted by number of repositories they've starred in descending order. ``` -TopN(Bitmap(frame="language", rowID=1), frame="stargazer", n=2) +TopN(Bitmap(frame="language", row=1), frame="stargazer", n=2) ``` Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]` @@ -439,7 +439,7 @@ Returns `[{"key": 1, "count": 2}, {"key": 2, "count": 1}]` **Spec:** ``` -Range(, , +Range(, , , ) ``` @@ -455,7 +455,7 @@ between the given `start` and `end` timestamps. When you set timestamp using SetBit, you will able to query all repositories that a user has starred within a date range. ``` -Range(frame="stargazer", rowID=1, start="2010-01-01T00:00", end="2017-03-02T03:00") +Range(frame="stargazer", row=1, start="2010-01-01T00:00", end="2017-03-02T03:00") ``` Returns `{{"attrs":{},"bits":[10]}` From 7ecd2ec79468ecd89ce5f76aa3378494c888dce0 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 23 Apr 2018 17:10:48 -0500 Subject: [PATCH 55/79] adjust references to RowID and ColumnID in the docs --- docs/administration.md | 6 +++--- docs/data-model.md | 2 +- docs/glossary.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index c9d932df0..190b5ca1f 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -48,9 +48,9 @@ On Mac OS X, `ulimit` does not behave predictably. [This blog post](https://blog #### Importing -The import API expects a csv of RowID,ColumnID's. +The import API expects a csv of rowID,columnID's. -When importing large datasets remember it is much faster to pre sort the data by RowID and then by ColumnID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results. +When importing large datasets remember it is much faster to pre sort the data by row ID and then by column ID in ascending order. You can use the `--sort` flag to do that. Also, avoid querying Pilosa until the import is complete, otherwise you will experience inconsistent results. ``` pilosa import --sort -i project -f stargazer project-stargazer.csv @@ -70,7 +70,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count #### Exporting -Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format RowID,ColumnID and sorted by column ID. +Exporting Data to csv can be performed on a live instance of Pilosa. You need to specify the Index, Frame, and View(default is standard). The API also expects the slice number, but the `pilosa export` sub command will export all slices within a Frame. The data will be in csv format rowID,columnID and sorted by columnID. ``` curl "http://localhost:10101/export?index=repository&frame=stargazer&slice=0&view=standard" \ --header "Accept: text/csv" diff --git a/docs/data-model.md b/docs/data-model.md index 693b4c97d..98750eda8 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -100,7 +100,7 @@ SetBit(frame="A", row=8, col=3, timestamp="2017-05-19T00:00") Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional bitmap indicating "not null". This means that a 16-bit integer will require 17 bitmaps: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null bitmap. Pilosa can evaluate `Sum` and `Range` queries on these BSI integers. -Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The 'rows' of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. +Internally Pilosa stores each BSI `field` as a `view` within a `frame`. The rows of the `view` are composed of the base-2 representation of the integer. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. For example, the following `SetFieldValue()` queries will result in the data described in the illustration below: diff --git a/docs/glossary.md b/docs/glossary.md index d222cb511..bd9293c3d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -24,7 +24,7 @@ nav = [] Fragment: A Fragment is the intersection of a [frame](#frame) and a [slice](#slice) in an [index](#index). -[Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. `RowID`s are namespaced by frame such that the same `RowID` in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. +[Frame](../data-model/#frame): Frames are used to group [rows](#row) into different categories. Row IDs are namespaced by frame such that the same row ID in a different frame refers to a different row. For [ranked](#topn) frames, rows are kept in sorted order within the frame. [Index](../data-model/#index): An Index is a top level container in Pilosa, analogous to a database in an RDBMS. Queries cannot operate across multiple indexes. @@ -62,6 +62,6 @@ nav = [] [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration/). -[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of `RowID`s, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). +[TopN](../query-language/#topn): A [PQL](#pql) query that returns a list of row IDs, sorted by the count of [bits](#bit) set in the [row](#row), within a specified [frame](#frame). [View](../data-model/#view): Views separate the different data layouts within a [Frame](#frame). The primary view is standard, which represents the typical [row](#row)/[column](#column) data. Time based frame views are automatically generated for each [time quantum](#time-quantum). Views are internally managed by Pilosa, and never exposed directly via the API. This simplifies the functional interface by separating it from the physical data representation. From ff8ab6930a66bfe2a55ce97484c7aea808db2189 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Tue, 24 Apr 2018 16:13:36 +0300 Subject: [PATCH 56/79] Run 32bits on CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 15c511e78..85e721329 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: install: - make install-dep install-statik vendor generate-statik script: + - GOARCH=386 make test - make test # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 after_success: From f9ff20689a29b631f131c4eefc018d5d5f177f7a Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Tue, 24 Apr 2018 08:44:58 -0500 Subject: [PATCH 57/79] remove holder.Peek, combine with HasData, move server logic Server initializing happens more in NewServer than Open now - expecting to continue this trend. goal was to remove remoteClient from Server (since it has a defaultClient) as well, but we'll have to refactor the client usage in fragment and frame first. --- cluster.go | 19 ++++++++++---- holder.go | 68 +++++++++++++++++++++++------------------------- holder_test.go | 29 ++++++++------------- server.go | 49 +++++++++++++++------------------- server/server.go | 1 + 5 files changed, 79 insertions(+), 87 deletions(-) diff --git a/cluster.go b/cluster.go index be668a6bc..bbf6f799e 100644 --- a/cluster.go +++ b/cluster.go @@ -17,7 +17,6 @@ package pilosa import ( "context" "encoding/binary" - "errors" "fmt" "hash/fnv" "io/ioutil" @@ -33,6 +32,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) @@ -899,7 +899,10 @@ func (c *Cluster) Open() error { } // Add the local node to the cluster. - c.AddNode(c.Node) + err := c.AddNode(c.Node) + if err != nil { + return errors.Wrap(err, "adding local node") + } // Start the EventReceiver. if err := c.EventReceiver.Start(c); err != nil { @@ -1684,13 +1687,15 @@ func (c *Cluster) nodeJoin(node *Node) error { // Only change to normal if there is no existing data. Otherwise, // the coordinator needs to wait to receive READY messages (nodeStates) // from remote nodes before setting the cluster to state NORMAL. - if !c.Holder.HasData() { + if ok, err := c.Holder.HasData(); !ok && err == nil { // If the result of the previous AddNode completed the joining of nodes // in the topology, then change the state to NORMAL. if c.haveTopologyAgreement() { return c.setStateAndBroadcast(ClusterStateNormal) } return nil + } else if err != nil { + return errors.Wrap(err, "checking if holder has data") } if c.haveTopologyAgreement() && c.allNodesReady() { @@ -1712,11 +1717,13 @@ func (c *Cluster) nodeJoin(node *Node) error { } // If the holder does not yet contain data, go ahead and add the node. - if !c.Holder.HasData() { + if ok, err := c.Holder.HasData(); !ok && err == nil { if err := c.AddNode(node); err != nil { return err } return c.setStateAndBroadcast(ClusterStateNormal) + } else if err != nil { + return errors.Wrap(err, "checking if holder has data2") } // If the cluster has data, we need to change to RESIZING and @@ -1770,11 +1777,13 @@ func (c *Cluster) nodeLeave(node *Node) error { } // If the holder does not yet contain data, go ahead and remove the node. - if !c.Holder.HasData() { + if ok, err := c.Holder.HasData(); !ok && err == nil { if err := c.RemoveNode(n); err != nil { return err } return c.setStateAndBroadcast(ClusterStateNormal) + } else if err != nil { + return errors.Wrap(err, "checking if holder has data") } // If the cluster has data then change state to RESIZING and diff --git a/holder.go b/holder.go index 2279a3e19..7649caeba 100644 --- a/holder.go +++ b/holder.go @@ -16,7 +16,6 @@ package pilosa import ( "context" - "errors" "fmt" "io/ioutil" "net/http" @@ -30,6 +29,7 @@ import ( "time" "github.com/pilosa/pilosa/internal" + "github.com/pkg/errors" uuid "github.com/satori/go.uuid" ) @@ -47,7 +47,6 @@ type Holder struct { // Indexes by name. indexes map[string]*Index - hasData bool // opened channel is closed once Open() completes. opened chan struct{} @@ -91,36 +90,6 @@ func NewHolder() *Holder { } } -// Peek reads the root data directory for the holder -// without actually loading any data into memory. -// HasData is returned, and h.hasData is set. -func (h *Holder) Peek() bool { - h.Logger.Printf("peek at holder path: %s", h.Path) - h.hasData = false - - // Open path to read all index directories. - f, err := os.Open(h.Path) - if err != nil { - return false - } - defer f.Close() - - fis, err := f.Readdir(0) - if err != nil { - return false - } - - for _, fi := range fis { - if !fi.IsDir() { - continue - } - h.hasData = true - break - } - - return h.hasData -} - // Open initializes the root data directory for the holder. func (h *Holder) Open() error { h.setFileLimit() @@ -198,10 +167,37 @@ func (h *Holder) Close() error { // HasData returns true if Holder contains at least one index. // This is used to determine if the rebalancing of data is necessary // when a node joins the cluster. -func (h *Holder) HasData() bool { - h.mu.RLock() - defer h.mu.RUnlock() - return h.hasData || len(h.indexes) > 0 +func (h *Holder) HasData() (bool, error) { + h.mu.Lock() + defer h.mu.Unlock() + if len(h.indexes) > 0 { + return true, nil + } + // Open path to read all index directories. + if _, err := os.Stat(h.Path); os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, errors.Wrap(err, "statting data dir") + } + + f, err := os.Open(h.Path) + if err != nil { + return false, errors.Wrap(err, "opening data dir") + } + defer f.Close() + + fis, err := f.Readdir(0) + if err != nil { + return false, errors.Wrap(err, "reading data dir") + } + + for _, fi := range fis { + if !fi.IsDir() { + continue + } + return true, nil + } + return false, nil } // MaxSlices returns MaxSlice map for all indexes. diff --git a/holder_test.go b/holder_test.go index 7053fa1f2..e819c6e94 100644 --- a/holder_test.go +++ b/holder_test.go @@ -283,26 +283,24 @@ func TestHolder_HasData(t *testing.T) { h := test.MustOpenHolder() defer h.Close() - if h.HasData() { - t.Fatal("expected HasData to return false") + if ok, err := h.HasData(); ok || err != nil { + t.Fatal("expected HasData to return false, no err, but", ok, err) } if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } - if !h.HasData() { - t.Fatal("expected HasData to return true") + if ok, err := h.HasData(); !ok || err != nil { + t.Fatal("expected HasData to return true, but ", ok, err) } }) t.Run("Peek", func(t *testing.T) { h := test.NewHolder() - if hasData := h.Peek(); hasData != false { - t.Fatal("expected Peek to return false") - } else if h.HasData() { - t.Fatal("expected HasData to return false") + if ok, err := h.HasData(); ok || err != nil { + t.Fatal("expected HasData to return false, no err, but", ok, err) } // Create an index directory to indicate data exists. @@ -310,24 +308,19 @@ func TestHolder_HasData(t *testing.T) { t.Fatal(err) } - if hasData := h.Peek(); hasData != true { - t.Fatal("expected Peek to return true") - } else if !h.HasData() { - t.Fatal("expected HasData to return true") + if ok, err := h.HasData(); !ok || err != nil { + t.Fatal("expected HasData to return true, no err, but", ok, err) } }) t.Run("Peek at missing directory", func(t *testing.T) { h := test.NewHolder() - // Ensure that hasData is false when trying to peek into - // a directory that doesn't exist. + // Ensure that hasData is false when dir doesn't exist. h.Path = "bad-path" - if hasData := h.Peek(); hasData != false { - t.Fatal("expected Peek to return false") - } else if h.HasData() { - t.Fatal("expected HasData to return false") + if ok, err := h.HasData(); ok || err != nil { + t.Fatal("expected HasData to return false, no err, but", ok, err) } }) } diff --git a/server.go b/server.go index d302a7d97..2b7903fb6 100644 --- a/server.go +++ b/server.go @@ -54,6 +54,7 @@ type Server struct { Holder *Holder Cluster *Cluster diagnostics *DiagnosticsCollector + executor *Executor // External handler *Handler @@ -161,7 +162,8 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { func OptServerRemoteClient(c *http.Client) ServerOption { return func(s *Server) error { - s.remoteClient = c + s.executor = NewExecutor(c) + s.defaultClient = NewInternalHTTPClientFromURI(nil, c) s.Cluster.RemoteClient = c return nil } @@ -231,11 +233,28 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.Cluster.Logger = s.logger s.Cluster.Holder = s.Holder - s.Cluster.RemoteClient = s.remoteClient + // update URI port with actual listener port. TODO this should probably be done outside of here. if s.URI.Port() == 0 { s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port)) } + + s.NodeID = s.LoadNodeID() + // Set Cluster Node. + node := &Node{ + ID: s.NodeID, + URI: s.URI, + IsCoordinator: s.Cluster.Coordinator == s.NodeID, + } + s.Cluster.Node = node + s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID)) + + s.executor.Holder = s.Holder + s.executor.Node = node + s.executor.Cluster = s.Cluster + s.executor.MaxWritesPerRequest = s.maxWritesPerRequest + s.handler.API.Executor = s.executor + return s, nil } @@ -248,33 +267,12 @@ func (s *Server) Open() error { } // Get or create NodeID. - s.NodeID = s.LoadNodeID() - - // Set Cluster Node. - node := &Node{ - ID: s.NodeID, - URI: s.URI, - IsCoordinator: s.Cluster.Coordinator == s.NodeID, - } - s.Cluster.Node = node // Append the NodeID tag to stats. - s.Holder.Stats = s.Holder.Stats.WithTags(fmt.Sprintf("NodeID:%s", s.NodeID)) - - // Peek at the holder to determine if there is data on disk. - // Don't actually load the data until after the Cluster - // management starts. - s.Holder.Peek() // Create default HTTP client - s.createDefaultClient(s.remoteClient) // Create executor for executing queries. - e := NewExecutor(s.remoteClient) - e.Holder = s.Holder - e.Node = node - e.Cluster = s.Cluster - e.MaxWritesPerRequest = s.maxWritesPerRequest // Cluster settings. s.Cluster.Broadcaster = s.Broadcaster @@ -287,7 +285,6 @@ func (s *Server) Open() error { s.handler.API.StatusHandler = s s.handler.API.URI = s.URI s.handler.API.Cluster = s.Cluster - s.handler.API.Executor = e // Initialize Holder. s.Holder.Broadcaster = s.Broadcaster @@ -750,10 +747,6 @@ func (s *Server) monitorRuntime() { } } -func (s *Server) createDefaultClient(remoteClient *http.Client) { - s.defaultClient = NewInternalHTTPClientFromURI(nil, remoteClient) -} - // CountOpenFiles on operating systems that support lsof. func CountOpenFiles() (int, error) { switch runtime.GOOS { diff --git a/server/server.go b/server/server.go index 7f5782211..8fd699ce5 100644 --- a/server/server.go +++ b/server/server.go @@ -305,6 +305,7 @@ func (m *Command) SetupNetworking() error { // Set Coordinator. if m.Config.Cluster.Coordinator || len(m.Config.Gossip.Seeds) == 0 { m.Server.Cluster.Coordinator = m.Server.NodeID + m.Server.Cluster.Node.IsCoordinator = true } gossipEventReceiver := gossip.NewGossipEventReceiver(m.logger) From 75bd93d89570608d49097b2db8ed6d86bf94fa51 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 27 Apr 2018 15:11:15 +0300 Subject: [PATCH 58/79] Fixes crash due to server.diagnostics.server not set --- server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server.go b/server.go index 2b7903fb6..7186e55cc 100644 --- a/server.go +++ b/server.go @@ -220,6 +220,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { logger: NopLogger, } + s.diagnostics.server = s for _, opt := range opts { err := opt(s) From 210c78a20a7bb4a9388494f7bc7f03f5fa1d528b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 27 Apr 2018 11:26:05 -0500 Subject: [PATCH 59/79] Use build matrix instead of separate commands to test different architectures --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 85e721329..46a25254c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ env: install: - make install-dep install-statik vendor generate-statik script: - - GOARCH=386 make test - make test # TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 after_success: @@ -28,6 +27,9 @@ deploy: matrix: allow_failures: - go: master + env: + - GOARCH=386 + - GOARCH=amd64 notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" From 0f8b630ab2bb8e6a8a652310d4202f932b22bd6f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 27 Apr 2018 11:43:31 -0500 Subject: [PATCH 60/79] Only run "deployment" (binary prerelease upload) for one build condition. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 46a25254c..5c7e8c70a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ deploy: on: all_branches: true go: "1.10" + condition: $GOARCH = amd64 matrix: allow_failures: - go: master From 52002f54556f656bc78c912614473f4cba44f5f0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 27 Apr 2018 14:51:54 -0500 Subject: [PATCH 61/79] Correct travis.ci matrix build config --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5c7e8c70a..3a9abe355 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,9 @@ env: global: # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - secure: "VnBFmFfBOrrf7ONLN9WpAFCcV8SEt5G5VPnnHv97TP7PlJG8LWR6k6O+vRJOvf8V4vDMfKCTDonwWLgbssVf3yygo3C8ZoftY2phehEkWGffCgsd9ML/YBNbGq4LYLSE5HKvBqrZjQaOrVby71BAsP8W7RhC6hqzFQ00M/z8dZVfwaQQFwew2eEcSxLEaaDFS8Wgc3/UuwxDRPBq6u3cCN5RxfB+q70HvGVq4TT+0dqS4eCvz688+Z0GIGYx9olNjh0F2Kc8R2Po0lnUNa0GiHrZ21zeQ1DxIK04QABrWWmjL4h+bx3VHNKPFR4GYSKDf+pj1kfaqbfrAg6rMAJdGejgoS+QyjhgCoN4d3qRp8s+1nrxtp0TvezEdjwyxt4quGHbP5TxWUszssbGhWqf4mx6OeJ8MmdTaJjfu0f3NWJXMycqT6J73WKORk4rHeIqF9CIdxdmcpkwYj8rk0TEMTPTsd7WA8w2HIDsCz/jQnRmEgLUiNnTAofYc/uUi/Wg/T2hllkp+oBDTzxk9NTelkqx8TJ0bDmYYL9JWUi1siFHTHiVYTJgyirSfGNpe61u8OLmT0Hak/D399IfL7qgFLlMXk8q92typfO2xEduq6G+8KygeqiOMSsOY+xcDvZf5xtcEihYd21vjtrxRSqFsup/o8DIxEurQnfXBx1B+WA=" - secure: "U4fpHWDVOG4viqZsiVgUDW7OW1JW60uPOZy0q9pfbs86iHvmZq0PaScsZ+YdlYaN2GETVr7endDf6DCcZs1PWfg0F6VQfkOXcShX8HVS9O58lUZA5tyvbDVql9DQs4PbnkZo+ktz+Z0YaXqq2RdtMDOUz4bgZwspLPMA14if+N6w0tqCFpB7bEtpptTGsdbIQPG1n07yvSeNmK4mvrEEs77tWmhulN5iilpOqhpIvD39bJvtCYVALuJpzLd/OjLTPV9l/fl+hJkMXSj+X5ilO1DHINAcCM648iEX2phXAIWmi0O0Rbg2cI4kV9T5ysOIw8ux+YCm9bZDGTCt+VGBW5Fg+Z5iaXXexyKYCGiHleOJ7kCj9kXxh2u8NiYVNgb19dGJV5/HgQ6pcGWjeVEqr8yY1546zMjpTX+SYGQF+XZe+uggEjeAsk53ueXa0pyZTrlrqSvR7BBtWPx47s/dTg2L19FQYv3XpGMxEXLw92RplExQKi1h7QgihRxFpjGgURHhrt7d9eiNiNqBt3ZsHjmh2AkXZHnaDjlgSnFFWaMqP3UtDBWIuO+2BMbZUJVfP+gpQGBZ4gtpUSmV2JDCHgZgX5OAnLD4usxh+ATQ4rvUXF/tf8nMqEKHlGKd8hxpYSyMX21BoqfSfY4/IA0ejVE9BITqlrvqewqkP1yxe7o=" + matrix: + - GOARCH=386 + - GOARCH=amd64 install: - make install-dep install-statik vendor generate-statik script: @@ -28,9 +31,6 @@ deploy: matrix: allow_failures: - go: master - env: - - GOARCH=386 - - GOARCH=amd64 notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" From 57ac2f40aaf1cd67467915e77cf35ff3a9010862 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Sat, 28 Apr 2018 09:09:03 -0500 Subject: [PATCH 62/79] fix nil client bug in monitorAntiEntropy (and test) also fix bug where broadcast_test tried to listen on localhost:10101 --- broadcast_test.go | 1 + server.go | 1 + server_test.go | 35 +++++++++++++++++++++++++++++++++++ test/pilosa.go | 34 +++++++++++++++++++++++++++------- 4 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 server_test.go diff --git a/broadcast_test.go b/broadcast_test.go index 187e50b4e..5654f7c28 100644 --- a/broadcast_test.go +++ b/broadcast_test.go @@ -57,6 +57,7 @@ func testMessageMarshal(t *testing.T, m proto.Message) { // Ensure that BroadcastReceiver can register a BroadcastHandler. func TestBroadcast_BroadcastReceiver(t *testing.T) { com := server.NewCommand(bytes.NewBuffer([]byte{}), ioutil.Discard, ioutil.Discard) + com.Config.Bind = "localhost:0" err := com.SetupServer() // this test shouldn't need to import pilosa/server just to set up the Server, but it really shouldn't need to setup the Server at all. The Server should not be the implementation of Broadcast* TODO if err != nil { t.Fatalf("setting up server: %v", err) diff --git a/server.go b/server.go index 7186e55cc..06260ae5d 100644 --- a/server.go +++ b/server.go @@ -163,6 +163,7 @@ func OptServerGCNotifier(gcn GCNotifier) ServerOption { func OptServerRemoteClient(c *http.Client) ServerOption { return func(s *Server) error { s.executor = NewExecutor(c) + s.remoteClient = c s.defaultClient = NewInternalHTTPClientFromURI(nil, c) s.Cluster.RemoteClient = c return nil diff --git a/server_test.go b/server_test.go new file mode 100644 index 000000000..9e10374d7 --- /dev/null +++ b/server_test.go @@ -0,0 +1,35 @@ +package pilosa_test + +import ( + "context" + "testing" + "time" + + "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/test" +) + +// TestMonitorAntiEntropy is a regression test which which caught a bug where +// pilosa.Server was not having its remoteClient field set by an option and so +// it was using a nil client in monitorAntiEntropy. +func TestMonitorAntiEntropy(t *testing.T) { + cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*1)) + client := cluster[1].Client() + err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + err = client.CreateFrame(context.Background(), "balh", "fralh", pilosa.FrameOptions{}) + if err != nil { + t.Fatalf("creating frame: %v", err) + } + + time.Sleep(time.Millisecond * 2) + for _, m := range cluster { + err := m.Close() + if err != nil { + t.Fatal(err) + } + } + +} diff --git a/test/pilosa.go b/test/pilosa.go index a390cd462..24838d5de 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -23,11 +23,13 @@ import ( "os" "strings" "testing" + "time" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/boltdb" "github.com/pilosa/pilosa/gossip" "github.com/pilosa/pilosa/server" + "github.com/pilosa/pilosa/toml" "github.com/pkg/errors" ) @@ -41,8 +43,17 @@ type Main struct { Stderr bytes.Buffer } +type MainOpt func(m *Main) error + +func OptAntiEntropyInterval(dur time.Duration) MainOpt { + return func(m *Main) error { + m.Command.Config.AntiEntropy.Interval = toml.Duration(dur) + return nil + } +} + // NewMain returns a new instance of Main with a temporary data directory and random port. -func NewMain() *Main { +func NewMain(opts ...MainOpt) *Main { path, err := ioutil.TempDir("", "pilosa-") if err != nil { panic(err) @@ -55,6 +66,15 @@ func NewMain() *Main { m.Command.Stdin = &m.Stdin m.Command.Stdout = &m.Stdout m.Command.Stderr = &m.Stderr + + for _, opt := range opts { + err := opt(m) + if err != nil { + panic(err) + } + + } + m.SetupServer() if testing.Verbose() { @@ -66,8 +86,8 @@ func NewMain() *Main { } // NewMainWithCluster returns a new instance of Main with clustering enabled. -func NewMainWithCluster(isCoordinator bool) *Main { - m := NewMain() +func NewMainWithCluster(isCoordinator bool, opts ...MainOpt) *Main { + m := NewMain(opts...) m.Config.Cluster.Disabled = false m.Config.Cluster.Coordinator = isCoordinator return m @@ -75,8 +95,8 @@ func NewMainWithCluster(isCoordinator bool) *Main { // MustRunMainWithCluster ruturns a running array of *Main where // all nodes are joined via memberlist (i.e. clustering enabled). -func MustRunMainWithCluster(t *testing.T, size int) []*Main { - ma, err := runMainWithCluster(size) +func MustRunMainWithCluster(t *testing.T, size int, opts ...MainOpt) []*Main { + ma, err := runMainWithCluster(size, opts...) if err != nil { t.Fatalf("new main array with cluster: %v", err) } @@ -85,7 +105,7 @@ func MustRunMainWithCluster(t *testing.T, size int) []*Main { // runMainWithCluster runs an array of *Main where all nodes are // joined via memberlist (i.e. clustering enabled). -func runMainWithCluster(size int) ([]*Main, error) { +func runMainWithCluster(size int, opts ...MainOpt) ([]*Main, error) { if size == 0 { return nil, errors.New("cluster must contain at least one node") } @@ -98,7 +118,7 @@ func runMainWithCluster(size int) ([]*Main, error) { var gossipSeeds = make([]string, size) for i := 0; i < size; i++ { - m := NewMainWithCluster(i == 0) + m := NewMainWithCluster(i == 0, opts...) m.Config.Cluster.Disabled = false gossipSeeds[i], err = m.RunWithTransport(gossipHost, gossipPort, gossipSeeds[:i]) From 5a56b33826691a027efe266dfd9ee69a32936374 Mon Sep 17 00:00:00 2001 From: Will Fitzgerald Date: Tue, 1 May 2018 05:45:19 -0400 Subject: [PATCH 63/79] fix minor spelling errors --- api.go | 2 +- gossip/gossip.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 700b21d25..02ffa91dc 100644 --- a/api.go +++ b/api.go @@ -788,7 +788,7 @@ func (api *API) DeleteView(ctx context.Context, indexName string, frameName stri // Delete the view. if err := f.DeleteView(viewName); err != nil { - // Ignore this error becuase views do not exist on all nodes due to slice distribution. + // Ignore this error because views do not exist on all nodes due to slice distribution. if err != ErrInvalidView { return err } diff --git a/gossip/gossip.go b/gossip/gossip.go index e4856ec7b..25dd40cc4 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -443,7 +443,7 @@ type Transport struct { // NewTransport returns a NetTransport based on the given host and port. // It will dynamically bind to a port if port is 0. -// This is useful for test cases where specifiying a port is not reasonable. +// This is useful for test cases where specifying a port is not reasonable. //func NewTransport(host string, port int) (*memberlist.NetTransport, error) { func NewTransport(host string, port int, logger *log.Logger) (*Transport, error) { // memberlist config From 331a4fc5b8d2818acbfa92bd23e7c673fb8783ad Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 2 May 2018 12:21:03 -0500 Subject: [PATCH 64/79] add NOTICE to binary distributions --- Dockerfile | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bf076e526..404be937f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,8 @@ LABEL maintainer "dev@pilosa.com" COPY --from=builder /go/bin/pilosa /pilosa +COPY NOTICE /NOTICE + EXPOSE 10101 VOLUME /data diff --git a/Makefile b/Makefile index cab932fba..e4d120659 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ build: vendor # Create a single release build under the build directory release-build: vendor $(MAKE) $(if $(DOCKER_BUILD),docker-)build FLAGS="-o build/pilosa-$(VERSION_ID)/pilosa" - cp LICENSE README.md build/pilosa-$(VERSION_ID) + cp NOTICE LICENSE README.md build/pilosa-$(VERSION_ID) tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ @echo Created release build: build/pilosa-$(VERSION_ID).tar.gz From 18d7664fc43cf2148a6c6c15557ade6574a289b7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 2 May 2018 15:04:12 -0500 Subject: [PATCH 65/79] Rename ID file to ".id" for consistency with existing data files. --- holder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holder.go b/holder.go index 7649caeba..d0a242ca5 100644 --- a/holder.go +++ b/holder.go @@ -524,7 +524,7 @@ func (h *Holder) setFileLimit() { } func (h *Holder) loadNodeID() (string, error) { - idPath := path.Join(h.Path, "ID") + idPath := path.Join(h.Path, ".id") nodeID := "" h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.Path, 0777); err != nil { From 85f90ac8fcf42a631c5490dba6500e2c37df419e Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 3 May 2018 16:11:43 -0500 Subject: [PATCH 66/79] Log time/version to startup log (Fixes #391) --- holder.go | 22 ++++++++++++++++++++++ server.go | 7 +++++++ 2 files changed, 29 insertions(+) diff --git a/holder.go b/holder.go index 7649caeba..49037d09f 100644 --- a/holder.go +++ b/holder.go @@ -547,6 +547,28 @@ func (h *Holder) loadNodeID() (string, error) { return nodeID, nil } +// Log startup time and version to $DATA_DIR/.startup.log +func (h *Holder) logStartup() error { + time, err := time.Now().MarshalText() + if err != nil { + return errors.Wrap(err, "creating timestamp") + } + logLine := fmt.Sprintf("%s\t%s\n", time, Version) + + f, err := os.OpenFile(h.Path+"/.startup.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return errors.Wrap(err, "opening startup log") + } + + defer f.Close() + + if _, err = f.WriteString(logLine); err != nil { + return errors.Wrap(err, "writing startup log") + } + + return nil +} + // HolderSyncer is an active anti-entropy tool that compares the local holder // with a remote holder based on block checksums and resolves differences. type HolderSyncer struct { diff --git a/server.go b/server.go index 06260ae5d..7617d7e69 100644 --- a/server.go +++ b/server.go @@ -17,6 +17,7 @@ package pilosa import ( "context" "fmt" + "log" "net" "net/http" "os" @@ -268,6 +269,12 @@ func (s *Server) Open() error { return errors.New("Must pass a listener option to NewServer") } + // Log startup + err := s.Holder.logStartup() + if err != nil { + log.Println(errors.Wrap(err, "logging startup")) + } + // Get or create NodeID. // Append the NodeID tag to stats. From e751945bf142cd53c80d2bfa930d00b945647e5f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 11:33:59 -0500 Subject: [PATCH 67/79] Add docs for upgrading Pilosa --- docs/administration.md | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index 190b5ca1f..eee32e419 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -92,7 +92,41 @@ The Pilosa server should support PQL versioning using HTTP headers. On each requ #### Upgrading -When upgrading, upgrade clients first, followed by server for all Minor and Patch level changes. +To upgrade Pilosa: + +1. First, upgrade the [client libraries](../client-libraries/) you are using in your application. Generally, a client version `X` will be compatible with the Pilosa server version `X` and earlier. For example, `python-pilosa 0.9.0` is compatible with both `pilosa 0.8.0` and `pilosa 0.9.0`. +2. Next, download the latest release from our [installation page](/docs/latest/installation/) or from the [release page on Github](https://github.com/pilosa/pilosa/releases). +3. Shut down the Pilosa cluster. A graceful shutdown should ensure that all data files are flushed to disk. +4. Make a backup of the [data directory](../configuration/#data-dir) on each cluster node. +5. Upgrade the Pilosa server binaries and any configuration changes. See the following sections on any version-specific changes you must make. +6. Start Pilosa. It is recommended to start the cluster coordinator node first, followed by any other nodes. + +##### Version 0.9 + +Pilosa v0.9 introduces a few compatibility changes that need to be addressed. + +**Configuration changes**: These changes need to occur during step 5 above: + +1. Cluster-resize capability eliminates the `hosts` setting. Now, cluster membership is determined by `gossip`. This is only a factor if you are running Pilosa as a cluster. +2. Gossip-based cluster membership requires you to set a single cluster node as a [coordinator](../configuration/#cluster-coordinator). Make sure only a single node has the `cluster.coordinator` flag set. +3. `gossip.seed` has been renamed [`gossip.seeds`](../configuration/#gossip-seeds) and takes multiple items. It is recommended that at least two nodes are specified as gossip seeds. + +**Data directory changes**: These changes also need to occur during step 5 above (while the cluster is shut down): + +Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called `topology-generator`: + +1. Observe the `cluster.hosts` configuration value in Pilosa v0.8. The ordering of the nodes in the config file is significant, as it determines shard (AKA slice) ownership. Pilosa v0.9 uses UUIDs for each node, and the ordering is alphabetical. +2. Install the `topology-generator`: `go get github.com/pilosa/upgrade-utils/v0.9/topology-generator`. +3. Run the `topology-generator`. There are two arguments: the number of nodes and the output directory. For this example, we'll assume a 3-node cluster and place the files in the current working directory: `topology-generator 3 .`. +4. This tool will generate a file, `topology`, and multiple id files, called `nodeX.id`, X being the node index position. +5. Copy the file `topology` into the data directories of every node in the cluster, naming it `.topology` (note the dot), e.g. `cp topology ~/.pilosa/.topology` or `scp topology node1:.pilosa/.topology`. +6. Copy the node ID files into the respective node data directories. For example, `node0.id` will be placed on the first node in the `cluster.hosts` list, with the name `.id`. For example: `scp node0.id node0:.pilosa/.id`. Again, it is very important that the ordering you give the nodes with these IDs matches the ordering you had in your existing `cluster.hosts` setting. + +**Application changes**: + +1. Row and column labels were deprecated in Pilosa v0.8, and removed in Pilosa v0.9. Make sure that your application does not attempt to use a custom row or column label, as they are no longer supported. +2. If your application relies on the implicit creation of [time quantums](../glossary/#time-quantum) by inheriting the time-quantum setting of the index, you must begin explicitly enabling the time quantum per-frame, as index-level time-quantums have been removed. +3. Inverse frames have been deprecated, removed from docs, and will be unsupported in the next release. ### Resizing the Cluster From 0afb95c1711b199501c282e731e82cbe9234fd66 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 12:33:20 -0500 Subject: [PATCH 68/79] Minor upgrading docs changes --- docs/administration.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index eee32e419..0ecbbed94 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -96,7 +96,7 @@ To upgrade Pilosa: 1. First, upgrade the [client libraries](../client-libraries/) you are using in your application. Generally, a client version `X` will be compatible with the Pilosa server version `X` and earlier. For example, `python-pilosa 0.9.0` is compatible with both `pilosa 0.8.0` and `pilosa 0.9.0`. 2. Next, download the latest release from our [installation page](/docs/latest/installation/) or from the [release page on Github](https://github.com/pilosa/pilosa/releases). -3. Shut down the Pilosa cluster. A graceful shutdown should ensure that all data files are flushed to disk. +3. Shut down the Pilosa cluster. 4. Make a backup of the [data directory](../configuration/#data-dir) on each cluster node. 5. Upgrade the Pilosa server binaries and any configuration changes. See the following sections on any version-specific changes you must make. 6. Start Pilosa. It is recommended to start the cluster coordinator node first, followed by any other nodes. @@ -105,13 +105,13 @@ To upgrade Pilosa: Pilosa v0.9 introduces a few compatibility changes that need to be addressed. -**Configuration changes**: These changes need to occur during step 5 above: +**Configuration changes**: These changes need to occur before starting Pilosa v0.9: 1. Cluster-resize capability eliminates the `hosts` setting. Now, cluster membership is determined by `gossip`. This is only a factor if you are running Pilosa as a cluster. 2. Gossip-based cluster membership requires you to set a single cluster node as a [coordinator](../configuration/#cluster-coordinator). Make sure only a single node has the `cluster.coordinator` flag set. 3. `gossip.seed` has been renamed [`gossip.seeds`](../configuration/#gossip-seeds) and takes multiple items. It is recommended that at least two nodes are specified as gossip seeds. -**Data directory changes**: These changes also need to occur during step 5 above (while the cluster is shut down): +**Data directory changes**: These changes need to occur while the cluster is shut down, before starting Pilosa v0.9: Pilosa v0.9 adds two new files to the data directory, an `.id` file and a `.topology` file. Due to the way Pilosa internally shards indices, upgrading a Pilosa cluster will result in data loss if an existing cluster is brought up without these files. New clusters will generate them automatically, but you may migrate an existing cluster by using a tool we called `topology-generator`: From 992c3e8bdda48572b0cc3eb67ac56c6a6a7e714b Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Thu, 26 Apr 2018 08:39:56 -0500 Subject: [PATCH 69/79] WIP - adding more locking to Cluster --- api.go | 7 +- cluster.go | 23 ++- cluster_test.go | 170 ++++++++--------- server/cluster_test.go | 4 +- test/pilosa.go | 7 +- utils_test.go | 425 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 538 insertions(+), 98 deletions(-) create mode 100644 utils_test.go diff --git a/api.go b/api.go index 02ffa91dc..9f7356cb8 100644 --- a/api.go +++ b/api.go @@ -1053,8 +1053,8 @@ func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode return nil, nil, errors.Wrap(err, "validate api method") } - oldNode = api.Cluster.nodeByID(api.Cluster.Coordinator) - newNode = api.Cluster.nodeByID(id) + oldNode = api.Cluster.NodeByID(api.Cluster.Coordinator) + newNode = api.Cluster.NodeByID(id) if newNode == nil { return nil, nil, errors.Wrap(ErrNodeIDNotExists, "getting new node") } @@ -1102,9 +1102,6 @@ func (api *API) ResizeAbort() error { return errors.Wrap(err, "validate api method") } - if !api.Cluster.IsCoordinator() { - return ErrNodeNotCoordinator - } err := api.Cluster.CompleteCurrentJob(ResizeJobStateAborted) return errors.Wrap(err, "complete current job") } diff --git a/cluster.go b/cluster.go index bbf6f799e..c9f5fd9a8 100644 --- a/cluster.go +++ b/cluster.go @@ -292,6 +292,12 @@ func (c *Cluster) CoordinatorNode() *Node { // IsCoordinator is true if this node is the coordinator. func (c *Cluster) IsCoordinator() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return c.isCoordinator() +} + +func (c *Cluster) isCoordinator() bool { return c.Coordinator == c.Node.ID } @@ -300,6 +306,8 @@ func (c *Cluster) IsCoordinator() bool { // will consider itself coordinator and update the other // nodes with its version of Cluster.Status. func (c *Cluster) SetCoordinator(n *Node) error { + c.mu.Lock() + defer c.mu.Unlock() // Verify that the new Coordinator value matches // this node. if c.Node.ID != n.ID { @@ -511,6 +519,12 @@ func (c *Cluster) Status() *internal.ClusterStatus { } } +func (c *Cluster) NodeByID(id string) *Node { + c.mu.RLock() + defer c.mu.RUnlock() + return c.nodeByID(id) +} + // nodeByID returns a node reference by ID. func (c *Cluster) nodeByID(id string) *Node { for _, n := range c.Nodes { @@ -1186,6 +1200,9 @@ func (c *Cluster) generateResizeJobByAction(nodeAction nodeAction) (*ResizeJob, func (c *Cluster) CompleteCurrentJob(state string) error { c.mu.Lock() defer c.mu.Unlock() + if !c.isCoordinator() { + return ErrNodeNotCoordinator + } if c.currentJob == nil { return ErrResizeNotRunning } @@ -1797,9 +1814,11 @@ func (c *Cluster) nodeLeave(node *Node) error { } func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { + c.mu.Lock() + defer c.mu.Unlock() c.Logger.Printf("merge cluster status: %v", cs) // Ignore status updates from self (coordinator). - if c.IsCoordinator() { + if c.isCoordinator() { return nil } @@ -1836,7 +1855,7 @@ func (c *Cluster) MergeClusterStatus(cs *internal.ClusterStatus) error { } } - c.SetState(cs.State) + c.setState(cs.State) c.markAsJoined() diff --git a/cluster_test.go b/cluster_test.go index 6c5d41829..b09820ac8 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa_test +package pilosa import ( "bytes" @@ -22,29 +22,27 @@ import ( "testing/quick" "github.com/davecgh/go-spew/spew" - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" ) // Ensure the cluster can fairly distribute partitions across the nodes. func TestCluster_Owners(t *testing.T) { - c := pilosa.Cluster{ - Nodes: []*pilosa.Node{ - {URI: test.NewURIFromHostPort("serverA", 1000)}, - {URI: test.NewURIFromHostPort("serverB", 1000)}, - {URI: test.NewURIFromHostPort("serverC", 1000)}, + c := Cluster{ + Nodes: []*Node{ + {URI: NewTestURIFromHostPort("serverA", 1000)}, + {URI: NewTestURIFromHostPort("serverB", 1000)}, + {URI: NewTestURIFromHostPort("serverC", 1000)}, }, - Hasher: test.NewModHasher(), + Hasher: NewTestModHasher(), ReplicaN: 2, } // Verify nodes are distributed. - if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[0], c.Nodes[1]}) { + if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*Node{c.Nodes[0], c.Nodes[1]}) { t.Fatalf("unexpected owners: %s", spew.Sdump(a)) } // Verify nodes go around the ring. - if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[2], c.Nodes[0]}) { + if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*Node{c.Nodes[2], c.Nodes[0]}) { t.Fatalf("unexpected owners: %s", spew.Sdump(a)) } } @@ -52,7 +50,7 @@ func TestCluster_Owners(t *testing.T) { // Ensure the partitioner can assign a fragment to a partition. func TestCluster_Partition(t *testing.T) { if err := quick.Check(func(index string, slice uint64, partitionN int) bool { - c := pilosa.NewCluster() + c := NewCluster() c.PartitionN = partitionN partitionID := c.Partition(index, slice) @@ -85,7 +83,7 @@ func TestHasher(t *testing.T) { {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, } { for i, v := range tt.bucket { - if got := pilosa.NewHasher().Hash(tt.key, i+1); got != v { + if got := NewHasher().Hash(tt.key, i+1); got != v { t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) } } @@ -94,8 +92,8 @@ func TestHasher(t *testing.T) { // Ensure OwnsSlices can find the actual slice list for node and index. func TestCluster_OwnsSlices(t *testing.T) { - c := test.NewCluster(5) - slices := c.OwnsSlices("test", 10, test.NewURIFromHostPort("host2", 0)) + c := NewTestCluster(5) + slices := c.OwnsSlices("test", 10, NewTestURIFromHostPort("host2", 0)) if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) { t.Fatalf("unexpected slices for node's index: %v", slices) @@ -104,7 +102,7 @@ func TestCluster_OwnsSlices(t *testing.T) { // Ensure ContainsSlices can find the actual slice list for node and index. func TestCluster_ContainsSlices(t *testing.T) { - c := test.NewCluster(5) + c := NewTestCluster(5) c.ReplicaN = 3 slices := c.ContainsSlices("test", 10, c.Nodes[2]) @@ -114,20 +112,20 @@ func TestCluster_ContainsSlices(t *testing.T) { } func TestCluster_Nodes(t *testing.T) { - uri0 := test.NewURIFromHostPort("node0", 0) - uri1 := test.NewURIFromHostPort("node1", 0) - uri2 := test.NewURIFromHostPort("node2", 0) - uri3 := test.NewURIFromHostPort("node3", 0) + uri0 := NewTestURIFromHostPort("node0", 0) + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) + uri3 := NewTestURIFromHostPort("node3", 0) - node0 := &pilosa.Node{ID: "node0", URI: uri0} - node1 := &pilosa.Node{ID: "node1", URI: uri1} - node2 := &pilosa.Node{ID: "node2", URI: uri2} - node3 := &pilosa.Node{ID: "node3", URI: uri3} + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + node3 := &Node{ID: "node3", URI: uri3} - nodes := []*pilosa.Node{node0, node1, node2} + nodes := []*Node{node0, node1, node2} t.Run("NodeIDs", func(t *testing.T) { - actual := pilosa.Nodes(nodes).IDs() + actual := Nodes(nodes).IDs() expected := []string{node0.ID, node1.ID, node2.ID} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) @@ -135,24 +133,24 @@ func TestCluster_Nodes(t *testing.T) { }) t.Run("Filter", func(t *testing.T) { - actual := pilosa.Nodes(pilosa.Nodes(nodes).Filter(nodes[1])).URIs() - expected := []pilosa.URI{uri0, uri2} + actual := Nodes(Nodes(nodes).Filter(nodes[1])).URIs() + expected := []URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) t.Run("FilterURI", func(t *testing.T) { - actual := pilosa.Nodes(pilosa.Nodes(nodes).FilterURI(uri1)).URIs() - expected := []pilosa.URI{uri0, uri2} + actual := Nodes(Nodes(nodes).FilterURI(uri1)).URIs() + expected := []URI{uri0, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } }) t.Run("Contains", func(t *testing.T) { - actualTrue := pilosa.Nodes(nodes).Contains(node1) - actualFalse := pilosa.Nodes(nodes).Contains(node3) + actualTrue := Nodes(nodes).Contains(node1) + actualFalse := Nodes(nodes).Contains(node3) if !reflect.DeepEqual(actualTrue, true) { t.Errorf("expected: %v, but got: %v", true, actualTrue) } @@ -162,9 +160,9 @@ func TestCluster_Nodes(t *testing.T) { }) t.Run("Clone", func(t *testing.T) { - clone := pilosa.Nodes(nodes).Clone() - actual := pilosa.Nodes(clone).URIs() - expected := []pilosa.URI{uri0, uri1, uri2} + clone := Nodes(nodes).Clone() + actual := Nodes(clone).URIs() + expected := []URI{uri0, uri1, uri2} if !reflect.DeepEqual(actual, expected) { t.Errorf("expected: %v, but got: %v", expected, actual) } @@ -172,16 +170,16 @@ func TestCluster_Nodes(t *testing.T) { } func TestCluster_Coordinator(t *testing.T) { - uri1 := test.NewURIFromHostPort("node1", 0) - uri2 := test.NewURIFromHostPort("node2", 0) + uri1 := NewTestURIFromHostPort("node1", 0) + uri2 := NewTestURIFromHostPort("node2", 0) - node1 := &pilosa.Node{ID: "node1", URI: uri1} - node2 := &pilosa.Node{ID: "node2", URI: uri2} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} - c1 := *pilosa.NewCluster() + c1 := *NewCluster() c1.Node = node1 c1.Coordinator = node1.ID - c2 := *pilosa.NewCluster() + c2 := *NewCluster() c2.Node = node2 c2.Coordinator = node1.ID @@ -195,17 +193,17 @@ func TestCluster_Coordinator(t *testing.T) { } func TestCluster_Topology(t *testing.T) { - c1 := test.NewCluster(1) // automatically creates Node{ID: "node0"} + c1 := NewTestCluster(1) // automatically creates Node{ID: "node0"} - uri0 := test.NewURIFromHostPort("host0", 0) - uri1 := test.NewURIFromHostPort("host1", 0) - uri2 := test.NewURIFromHostPort("host2", 0) - invalid := test.NewURIFromHostPort("invalid", 0) + uri0 := NewTestURIFromHostPort("host0", 0) + uri1 := NewTestURIFromHostPort("host1", 0) + uri2 := NewTestURIFromHostPort("host2", 0) + invalid := NewTestURIFromHostPort("invalid", 0) - node0 := &pilosa.Node{ID: "node0", URI: uri0} - node1 := &pilosa.Node{ID: "node1", URI: uri1} - node2 := &pilosa.Node{ID: "node2", URI: uri2} - nodeinvalid := &pilosa.Node{ID: "nodeinvalid", URI: invalid} + node0 := &Node{ID: "node0", URI: uri0} + node1 := &Node{ID: "node1", URI: uri1} + node2 := &Node{ID: "node2", URI: uri2} + nodeinvalid := &Node{ID: "nodeinvalid", URI: invalid} t.Run("AddNode", func(t *testing.T) { err := c1.AddNode(node1) @@ -243,7 +241,7 @@ func TestCluster_Topology(t *testing.T) { func TestCluster_ResizeStates(t *testing.T) { t.Run("Single node, no data", func(t *testing.T) { - tc := test.NewTestCluster(1) + tc := NewClusterCluster(1) // Open TestCluster. if err := tc.Open(); err != nil { @@ -253,11 +251,11 @@ func TestCluster_ResizeStates(t *testing.T) { node := tc.Clusters[0] // Ensure that node comes up in state NORMAL. - if node.State() != pilosa.ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State()) + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) } - expectedTop := &pilosa.Topology{ + expectedTop := &Topology{ NodeIDs: []string{node.Node.ID}, } @@ -273,13 +271,13 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Single node, in topology", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) node := tc.Clusters[0] // write topology to data file - top := &pilosa.Topology{ + top := &Topology{ NodeIDs: []string{node.Node.ID}, } tc.WriteTopology(node.Path, top) @@ -290,8 +288,8 @@ func TestCluster_ResizeStates(t *testing.T) { } // Ensure that node comes up in state NORMAL. - if node.State() != pilosa.ClusterStateNormal { - t.Errorf("expected state: %v, but got: %v", pilosa.ClusterStateNormal, node.State()) + if node.State() != ClusterStateNormal { + t.Errorf("expected state: %v, but got: %v", ClusterStateNormal, node.State()) } // Close TestCluster. @@ -301,13 +299,13 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Single node, not in topology", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) node := tc.Clusters[0] // write topology to data file - top := &pilosa.Topology{ + top := &Topology{ NodeIDs: []string{"some-other-host"}, } tc.WriteTopology(node.Path, top) @@ -326,7 +324,7 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Multiple nodes, no data", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) // Open TestCluster. @@ -340,13 +338,13 @@ func TestCluster_ResizeStates(t *testing.T) { node1 := tc.Clusters[1] // Ensure that nodes comes up in state NORMAL. - if node0.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State()) - } else if node1.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State()) + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) } - expectedTop := &pilosa.Topology{ + expectedTop := &Topology{ NodeIDs: []string{node0.Node.ID, node1.Node.ID}, } @@ -364,12 +362,12 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Multiple nodes, in/not in topology", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) node0 := tc.Clusters[0] // write topology to data file - top := &pilosa.Topology{ + top := &Topology{ NodeIDs: []string{"node0", "node2"}, } tc.WriteTopology(node0.Path, top) @@ -380,8 +378,8 @@ func TestCluster_ResizeStates(t *testing.T) { } // Ensure that node is in state STARTING before the other node joins. - if node0.State() != pilosa.ClusterStateStarting { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateStarting, node0.State()) + if node0.State() != ClusterStateStarting { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateStarting, node0.State()) } // Expect an error by adding a node not in the topology. @@ -395,10 +393,10 @@ func TestCluster_ResizeStates(t *testing.T) { node2 := tc.Clusters[2] // Ensure that node comes up in state NORMAL. - if node0.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State()) - } else if node2.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node2.State()) + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node2.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node2.State()) } // Close TestCluster. @@ -408,7 +406,7 @@ func TestCluster_ResizeStates(t *testing.T) { }) t.Run("Multiple nodes, with data", func(t *testing.T) { - tc := test.NewTestCluster(0) + tc := NewClusterCluster(0) tc.AddNode(false) node0 := tc.Clusters[0] @@ -418,20 +416,20 @@ func TestCluster_ResizeStates(t *testing.T) { } // Add Bit Data to node0. - if err := tc.CreateFrame("i", "f", pilosa.FrameOptions{}); err != nil { + if err := tc.CreateFrame("i", "f", FrameOptions{}); err != nil { t.Fatal(err) } tc.SetBit("i", "f", "standard", 1, 101, nil) tc.SetBit("i", "f", "standard", 1, 1300000, nil) // Add Field Data to node0. - if err := tc.CreateFrame("i", "fields", pilosa.FrameOptions{ + if err := tc.CreateFrame("i", "fields", FrameOptions{ InverseEnabled: false, - //CacheType: pilosa.CacheTypeNone, - Fields: []*pilosa.Field{ + //CacheType: CacheTypeNone, + Fields: []*Field{ { Name: "fld0", - Type: pilosa.FieldTypeInt, + Type: FieldTypeInt, Min: -100, Max: 100, }, @@ -461,13 +459,13 @@ func TestCluster_ResizeStates(t *testing.T) { node1 := tc.Clusters[1] // Ensure that nodes come up in state NORMAL. - if node0.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node0 state: %v, but got: %v", pilosa.ClusterStateNormal, node0.State()) - } else if node1.State() != pilosa.ClusterStateNormal { - t.Errorf("expected node1 state: %v, but got: %v", pilosa.ClusterStateNormal, node1.State()) + if node0.State() != ClusterStateNormal { + t.Errorf("expected node0 state: %v, but got: %v", ClusterStateNormal, node0.State()) + } else if node1.State() != ClusterStateNormal { + t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State()) } - expectedTop := &pilosa.Topology{ + expectedTop := &Topology{ NodeIDs: []string{node0.Node.ID, node1.Node.ID}, } @@ -510,7 +508,7 @@ func TestCluster_ResizeStates(t *testing.T) { // Ensures that coordinator can be changed. func TestCluster_UpdateCoordinator(t *testing.T) { t.Run("UpdateCoordinator", func(t *testing.T) { - c := test.NewCluster(2) + c := NewTestCluster(2) oldNode := c.Nodes[0] newNode := c.Nodes[1] diff --git a/server/cluster_test.go b/server/cluster_test.go index 1a64ce305..326753a3d 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -396,7 +396,7 @@ func TestCluster_GossipMembership(t *testing.T) { var eg errgroup.Group eg.Go(func() error { // Pass invalid seed as first in list - _, err = m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed}) + _, err := m1.RunWithTransport("localhost", 0, []string{"http://localhost:8765", seed}) if err != nil { return err } @@ -409,7 +409,7 @@ func TestCluster_GossipMembership(t *testing.T) { eg.Go(func() error { // Pass invalid seed as last in list - _, err = m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"}) + _, err := m2.RunWithTransport("localhost", 0, []string{seed, "http://localhost:8765"}) if err != nil { return err } diff --git a/test/pilosa.go b/test/pilosa.go index 24838d5de..f6e3a3457 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -66,7 +66,6 @@ func NewMain(opts ...MainOpt) *Main { m.Command.Stdin = &m.Stdin m.Command.Stdout = &m.Stdout m.Command.Stderr = &m.Stderr - for _, opt := range opts { err := opt(m) if err != nil { @@ -74,8 +73,10 @@ func NewMain(opts ...MainOpt) *Main { } } - - m.SetupServer() + err = m.SetupServer() + if err != nil { + panic(err) + } if testing.Verbose() { m.Command.Stdout = io.MultiWriter(os.Stdout, m.Command.Stdout) diff --git a/utils_test.go b/utils_test.go new file mode 100644 index 000000000..7bc17d424 --- /dev/null +++ b/utils_test.go @@ -0,0 +1,425 @@ +package pilosa + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "path/filepath" + "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" +) + +// NewCluster returns a cluster with n nodes and uses a mod-based hasher. +func NewTestCluster(n int) *Cluster { + path, err := ioutil.TempDir("", "pilosa-cluster-") + if err != nil { + panic(err) + } + + c := NewCluster() + c.ReplicaN = 1 + c.Hasher = NewTestModHasher() + c.Path = path + c.Topology = NewTopology() + + for i := 0; i < n; i++ { + c.Nodes = append(c.Nodes, &Node{ + ID: fmt.Sprintf("node%d", i), + URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), + }) + } + + c.Node = c.Nodes[0] + c.Coordinator = c.Nodes[0].ID + c.SetState(ClusterStateNormal) + + return c +} + +// NewURI is a test URI creator that intentionally swallows errors. +func NewTestURI(scheme, host string, port uint16) URI { + uri := DefaultURI() + uri.SetScheme(scheme) + uri.SetHost(host) + uri.SetPort(port) + return *uri +} + +func NewTestURIFromHostPort(host string, port uint16) URI { + uri := DefaultURI() + uri.SetHost(host) + uri.SetPort(port) + return *uri +} + +// ModHasher represents a simple, mod-based hashing. +type TestModHasher struct{} + +// NewModHasher returns a new instance of ModHasher with n buckets. +func NewTestModHasher() *TestModHasher { return &TestModHasher{} } + +func (*TestModHasher) Hash(key uint64, n int) int { return int(key) % n } + +// ClusterCluster represents a cluster of test nodes, each of which +// has a Cluster. +type ClusterCluster struct { + Clusters []*Cluster + + common *commonClusterSettings + + mu sync.RWMutex + resizing bool + resizeDone chan struct{} +} + +type commonClusterSettings struct { + Nodes []*Node +} + +func (t *ClusterCluster) CreateIndex(name string) error { + for _, c := range t.Clusters { + if _, err := c.Holder.CreateIndexIfNotExists(name, IndexOptions{}); err != nil { + return err + } + } + return nil +} + +func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) error { + for _, c := range t.Clusters { + idx, err := c.Holder.CreateIndexIfNotExists(index, IndexOptions{}) + if err != nil { + return err + } + if _, err := idx.CreateFrame(frame, opt); err != nil { + return err + } + } + return nil +} +func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { + // Determine which node should receive the SetBit. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := colID / SliceWidth + nodes := c0.SliceNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByID(node.ID) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetBit(view, rowID, colID, x) + if err != nil { + return err + } + } + + return nil +} + +func (t *ClusterCluster) SetFieldValue(index, frame string, columnID uint64, name string, value int64) error { + // Determine which node should receive the SetFieldValue. + c0 := t.Clusters[0] // use the first node's cluster to determine slice location. + slice := columnID / SliceWidth + nodes := c0.SliceNodes(index, slice) + + for _, node := range nodes { + c := t.clusterByID(node.ID) + if c == nil { + continue + } + f := c.Holder.Frame(index, frame) + if f == nil { + return fmt.Errorf("index/frame does not exist: %s/%s", index, frame) + } + _, err := f.SetFieldValue(columnID, name, value) + if err != nil { + return err + } + } + + return nil +} + +func (t *ClusterCluster) clusterByID(id string) *Cluster { + for _, c := range t.Clusters { + if c.Node.ID == id { + return c + } + } + return nil +} + +// AddNode adds a node to the cluster and (potentially) starts a resize job. +func (t *ClusterCluster) AddNode(saveTopology bool) error { + id := len(t.Clusters) + + c, err := t.addCluster(id, saveTopology) + if err != nil { + return err + } + + // Send NodeJoin event to coordinator. + if id > 0 { + coord := t.Clusters[0] + ev := &NodeEvent{ + Event: NodeJoin, + Node: c.Node, + } + + if err := coord.ReceiveEvent(ev); err != nil { + return err + } + + // Wait for the AddNode job to finish. + if c.State() != ClusterStateNormal { + t.resizeDone = make(chan struct{}) + t.mu.Lock() + t.resizing = true + t.mu.Unlock() + <-t.resizeDone + } + } + + return nil +} + +// WriteTopology writes the given topology to disk. +func (t *ClusterCluster) WriteTopology(path string, top *Topology) error { + if buf, err := proto.Marshal(top.Encode()); err != nil { + return err + } else if err := ioutil.WriteFile(filepath.Join(path, ".topology"), buf, 0666); err != nil { + return err + } + return nil +} + +func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*Cluster, error) { + + id := fmt.Sprintf("node%d", i) + uri := NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)) + + node := &Node{ + ID: id, + URI: uri, + } + + // add URI to common + //t.common.NodeIDs = append(t.common.NodeIDs, id) + //sort.Sort(t.common.NodeIDs) + + // add node to common + t.common.Nodes = append(t.common.Nodes, node) + + // create node-specific temp directory + path, err := ioutil.TempDir("", fmt.Sprintf("pilosa-cluster-node-%d-", i)) + if err != nil { + return nil, err + } + + // holder + h := NewHolder() + h.Path = path + + // cluster + c := NewCluster() + c.ReplicaN = 1 + c.Hasher = NewTestModHasher() + c.Path = path + c.Topology = NewTopology() + c.Holder = h + c.MemberSet = NewStaticMemberSet(c.Nodes) + c.Node = node + c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator + c.Broadcaster = t + + // add nodes + if saveTopology { + for _, n := range t.common.Nodes { + c.AddNode(n) + } + } + + // Add this node to the ClusterCluster. + t.Clusters = append(t.Clusters, c) + + return c, nil +} + +// NewClusterCluster returns a new instance of test.Cluster. +func NewClusterCluster(n int) *ClusterCluster { + + tc := &ClusterCluster{ + common: &commonClusterSettings{}, + } + + // add clusters + for i := 0; i < n; i++ { + _, err := tc.addCluster(i, true) + if err != nil { + panic(err) + } + } + return tc +} + +// SetState sets the state of the cluster on each node. +func (t *ClusterCluster) SetState(state string) { + for _, c := range t.Clusters { + c.SetState(state) + } +} + +// Open opens all clusters in the test cluster. +func (t *ClusterCluster) Open() error { + for _, c := range t.Clusters { + if err := c.Open(); err != nil { + return err + } + if err := c.Holder.Open(); err != nil { + return err + } + if err := c.SetNodeState(NodeStateReady); err != nil { + return err + } + } + + // Start the listener on the coordinator. + if len(t.Clusters) == 0 { + return nil + } + t.Clusters[0].ListenForJoins() + + return nil +} + +// Close closes all clusters in the test cluster. +func (t *ClusterCluster) Close() error { + for _, c := range t.Clusters { + err := c.Close() + if err != nil { + return err + } + } + return nil +} + +// ClusterCluster implements Broadcaster interface. + +// SendSync is a test implemenetation of Broadcaster SendSync method. +func (t *ClusterCluster) SendSync(pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ClusterStatus: + // Apply the send message to all nodes (except the coordinator). + for _, c := range t.Clusters { + c.MergeClusterStatus(obj) + } + t.mu.RLock() + if obj.State == ClusterStateNormal && t.resizing { + close(t.resizeDone) + } + t.mu.RUnlock() + } + + return nil +} + +// SendAsync is a test implemenetation of Broadcaster SendAsync method. +func (t *ClusterCluster) SendAsync(pb proto.Message) error { + return nil +} + +// SendTo is a test implemenetation of Broadcaster SendTo method. +func (t *ClusterCluster) SendTo(to *Node, pb proto.Message) error { + switch obj := pb.(type) { + case *internal.ResizeInstruction: + err := t.FollowResizeInstruction(obj) + if err != nil { + return err + } + case *internal.ResizeInstructionComplete: + coord := t.clusterByID(to.ID) + go coord.MarkResizeInstructionComplete(obj) + } + return nil +} + +// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. +func (t *ClusterCluster) FollowResizeInstruction(instr *internal.ResizeInstruction) error { + + // Prepare the return message. + complete := &internal.ResizeInstructionComplete{ + JobID: instr.JobID, + Node: instr.Node, + Error: "", + } + + // Stop processing on any error. + if err := func() error { + + // figure out which node it was meant for, then call the operation on that cluster + // basically need to mimic this: client.RetrieveSliceFromURI(context.Background(), src.Index, src.Frame, src.View, src.Slice, srcURI) + instrNode := DecodeNode(instr.Node) + destCluster := t.clusterByID(instrNode.ID) + + // Sync the schema received in the resize instruction. + if err := destCluster.Holder.ApplySchema(instr.Schema); err != nil { + return err + } + + for _, src := range instr.Sources { + srcNode := DecodeNode(src.Node) + srcCluster := t.clusterByID(srcNode.ID) + + srcFragment := srcCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + destFragment := destCluster.Holder.Fragment(src.Index, src.Frame, src.View, src.Slice) + if destFragment == nil { + // Create fragment on destination if it doesn't exist. + f := destCluster.Holder.Frame(src.Index, src.Frame) + v := f.View(src.View) + var err error + destFragment, err = v.CreateFragmentIfNotExists(src.Slice) + if err != nil { + return err + } + } + + buf := bytes.NewBuffer(nil) + + bw := bufio.NewWriter(buf) + br := bufio.NewReader(buf) + + // Get the fragment from source. + if _, err := srcFragment.WriteTo(bw); err != nil { + return err + } + + // Flush the bufio.buf to the io.Writer (buf). + bw.Flush() + + // Write data to destination. + if _, err := destFragment.ReadFrom(br); err != nil { + return err + } + } + + return nil + }(); err != nil { + complete.Error = err.Error() + } + + node := DecodeNode(instr.Coordinator) + if err := t.SendTo(node, complete); err != nil { + return err + } + + return nil +} From 2ef78af0ca86af9336cf61d1b7d88ab7eec692ce Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 26 Apr 2018 10:42:50 -0500 Subject: [PATCH 70/79] adjusted protection around cluster coordinator mutations --- cluster.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index c9f5fd9a8..c7285d17a 100644 --- a/cluster.go +++ b/cluster.go @@ -315,7 +315,7 @@ func (c *Cluster) SetCoordinator(n *Node) error { } // Update IsCoordinator on all nodes (locally). - _ = c.UpdateCoordinator(n) + _ = c.updateCoordinator(n) // Send the update coordinator message to all nodes. err := c.Broadcaster.SendSync( @@ -335,6 +335,12 @@ func (c *Cluster) SetCoordinator(n *Node) error { // to true, and sets all other nodes to false. Returns true if the value // changed. func (c *Cluster) UpdateCoordinator(n *Node) bool { + c.mu.Lock() + defer c.mu.Unlock() + _ = c.updateCoordinator(n) +} + +func (c *Cluster) updateCoordinator(n *Node) bool { var changed bool if c.Coordinator != n.ID { c.Coordinator = n.ID From 4a1d15ec9dd235d39628488947ec1d337f23aca2 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 26 Apr 2018 11:43:39 -0500 Subject: [PATCH 71/79] missed a file on last commit --- cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index c7285d17a..b02bd50d2 100644 --- a/cluster.go +++ b/cluster.go @@ -337,7 +337,7 @@ func (c *Cluster) SetCoordinator(n *Node) error { func (c *Cluster) UpdateCoordinator(n *Node) bool { c.mu.Lock() defer c.mu.Unlock() - _ = c.updateCoordinator(n) + return c.updateCoordinator(n) } func (c *Cluster) updateCoordinator(n *Node) bool { From 985b56f50b6f0f660b1f86d6810fb967d19c2d03 Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Fri, 27 Apr 2018 10:20:24 -0500 Subject: [PATCH 72/79] fix utils_test comments --- utils_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils_test.go b/utils_test.go index 7bc17d424..05afeac19 100644 --- a/utils_test.go +++ b/utils_test.go @@ -13,7 +13,7 @@ import ( "github.com/pilosa/pilosa/internal" ) -// NewCluster returns a cluster with n nodes and uses a mod-based hasher. +// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. func NewTestCluster(n int) *Cluster { path, err := ioutil.TempDir("", "pilosa-cluster-") if err != nil { @@ -40,7 +40,7 @@ func NewTestCluster(n int) *Cluster { return c } -// NewURI is a test URI creator that intentionally swallows errors. +// NewTestURI is a test URI creator that intentionally swallows errors. func NewTestURI(scheme, host string, port uint16) URI { uri := DefaultURI() uri.SetScheme(scheme) @@ -59,13 +59,14 @@ func NewTestURIFromHostPort(host string, port uint16) URI { // ModHasher represents a simple, mod-based hashing. type TestModHasher struct{} -// NewModHasher returns a new instance of ModHasher with n buckets. +// NewTestModHasher returns a new instance of ModHasher with n buckets. func NewTestModHasher() *TestModHasher { return &TestModHasher{} } func (*TestModHasher) Hash(key uint64, n int) int { return int(key) % n } // ClusterCluster represents a cluster of test nodes, each of which // has a Cluster. +// ClusterCluster implements Broadcaster interface. type ClusterCluster struct { Clusters []*Cluster @@ -101,6 +102,7 @@ func (t *ClusterCluster) CreateFrame(index, frame string, opt FrameOptions) erro } return nil } + func (t *ClusterCluster) SetBit(index, frame, view string, rowID, colID uint64, x *time.Time) error { // Determine which node should receive the SetBit. c0 := t.Clusters[0] // use the first node's cluster to determine slice location. @@ -312,8 +314,6 @@ func (t *ClusterCluster) Close() error { return nil } -// ClusterCluster implements Broadcaster interface. - // SendSync is a test implemenetation of Broadcaster SendSync method. func (t *ClusterCluster) SendSync(pb proto.Message) error { switch obj := pb.(type) { From 84ed1953bd60a8cdb5eb9324ac79d3ab68746eb7 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 2 May 2018 11:52:35 -0500 Subject: [PATCH 73/79] Update changelog --- CHANGELOG.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09562dc06..ed2baf7af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## Unreleased +## [v0.9.0] - 2018-05-02 + +This version contains 183 contribution from 12 contributors. There are 138 files changed, 17,084 insertions, and 7,482 deletions. ### Added @@ -23,7 +25,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013)) - Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) - Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972)) +- Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) +- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972)), ([#1215](https://github.com/pilosa/pilosa/pull/1215)), ([#1213](https://github.com/pilosa/pilosa/pull/1213)), ([#1224](https://github.com/pilosa/pilosa/pull/1224)) ### Changed @@ -42,12 +45,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) - Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) - Deprecated RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) +- Refactoring pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) +- Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) +- Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) ### Removed - Remove the Gossip stutter from memberlist-related config options ([#1171](https://github.com/pilosa/pilosa/pull/1171)) - Remove old GossipPort and GossipSeed config options ([#1142](https://github.com/pilosa/pilosa/pull/1142)) - Remove cluster type `http` from docs ([#1130](https://github.com/pilosa/pilosa/pull/1130)) +- Remove holder.Peek, combine with HasData, move server logic ([#1226](https://github.com/pilosa/pilosa/pull/1226)) +- Remove PATCH frame endpoint ([#1222](https://github.com/pilosa/pilosa/pull/1222)) +- Remove Index.MergeSchemas() method ([#1219](https://github.com/pilosa/pilosa/pull/1219)) +- Remove references to Input Definition from the docs ([#1212](https://github.com/pilosa/pilosa/pull/1212)) +- Remove Index.TimeQuantum ([#1209](https://github.com/pilosa/pilosa/pull/1209)) +- Remove SecurityManager. Implement api restrictions in api package. ([#1207](https://github.com/pilosa/pilosa/pull/1207)) ### Fixed @@ -60,6 +72,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Fix count/bitmap mismatch bug ([#1084](https://github.com/pilosa/pilosa/pull/1084)) - Fix edge case with Range() calls outside field Min/Max. Fixes #876. ([#979](https://github.com/pilosa/pilosa/pull/979)) - Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) +- Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) +- Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) ### Performance From 46a929e1d64701e42d47088f2fe60e05911a1d94 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 15:49:03 -0500 Subject: [PATCH 74/79] More updates for v0.9 branch --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed2baf7af..f9671e748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [v0.9.0] - 2018-05-02 +## [v0.9.0] - 2018-05-04 This version contains 183 contribution from 12 contributors. There are 138 files changed, 17,084 insertions, and 7,482 deletions. @@ -22,11 +22,12 @@ This version contains 183 contribution from 12 contributors. There are 138 files - Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) - Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) - Adds support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) -- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013)) +- Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) - Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) - Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) - Implement Min/Max BSI queries ([#1191](https://github.com/pilosa/pilosa/pull/1191)) -- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972)), ([#1215](https://github.com/pilosa/pilosa/pull/1215)), ([#1213](https://github.com/pilosa/pilosa/pull/1213)), ([#1224](https://github.com/pilosa/pilosa/pull/1224)) +- Log time/version to startup log ([#1246](https://github.com/pilosa/pilosa/pull/1246)) +- Documentation improvements ([#1135](https://github.com/pilosa/pilosa/pull/1135), [#1154](https://github.com/pilosa/pilosa/pull/1154), [#1091](https://github.com/pilosa/pilosa/pull/1091), [#1108](https://github.com/pilosa/pilosa/pull/1108), [#1087](https://github.com/pilosa/pilosa/pull/1087), [#1086](https://github.com/pilosa/pilosa/pull/1086), [#1026](https://github.com/pilosa/pilosa/pull/1026), [#1022](https://github.com/pilosa/pilosa/pull/1022), [#1007](https://github.com/pilosa/pilosa/pull/1007), [#981](https://github.com/pilosa/pilosa/pull/981), [#901](https://github.com/pilosa/pilosa/pull/901), [#972](https://github.com/pilosa/pilosa/pull/972), [#1215](https://github.com/pilosa/pilosa/pull/1215), [#1213](https://github.com/pilosa/pilosa/pull/1213), [#1224](https://github.com/pilosa/pilosa/pull/1224), [#1250](https://github.com/pilosa/pilosa/pull/1250)) ### Changed @@ -48,6 +49,7 @@ This version contains 183 contribution from 12 contributors. There are 138 files - Refactoring pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) - Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) - Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) +- Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) ### Removed @@ -74,6 +76,7 @@ This version contains 183 contribution from 12 contributors. There are 138 files - Bind the handler to all interfaces (0.0.0.0) in Dockerfile. Fixes #977. ([#980](https://github.com/pilosa/pilosa/pull/980)) - Fix nil client bug in monitorAntiEntropy (and test) ([#1233](https://github.com/pilosa/pilosa/pull/1233)) - Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) +- Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) ### Performance From 995a90ac9d3b1f6578f19c77b99d917cd5357c6d Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 16:22:00 -0500 Subject: [PATCH 75/79] Update pilosa version in installation docs --- docs/installation.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index db0828b2f..a0632c34c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -74,19 +74,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.8.3-darwin-amd64.tar.gz + tar xfz pilosa-v0.9.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.8.3-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.9.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -234,19 +234,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.8.3/pilosa-v0.8.3-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v0.9.0/pilosa-v0.9.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v0.8.3-linux-amd64.tar.gz + tar xfz pilosa-v0.9.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v0.8.3-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v0.9.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: From 3529043fbcc4df1c29b5026138f055e9fc05bf57 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 16:22:08 -0500 Subject: [PATCH 76/79] Update Go version --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 404be937f..0392dbe87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10 as builder +FROM golang:1.10.2 as builder COPY . /go/src/github.com/pilosa/pilosa/ From 4454bedb3d14f1df9bc7a1c197a855f557f5e451 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 16:28:19 -0500 Subject: [PATCH 77/79] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9671e748..2a0dc777c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [v0.9.0] - 2018-05-04 -This version contains 183 contribution from 12 contributors. There are 138 files changed, 17,084 insertions, and 7,482 deletions. +This version contains 188 contribution from 12 contributors. There are 141 files changed, 17,832 insertions, and 7,503 deletions. ### Added From 76ee46bd0d265eb4cd87e24797d9860a9b7bebf3 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 17:03:43 -0500 Subject: [PATCH 78/79] Add note about upgrading to v0.9 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a0dc777c..32c84b835 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). This version contains 188 contribution from 12 contributors. There are 141 files changed, 17,832 insertions, and 7,503 deletions. +*Please see special [upgrading instructions](https://www.pilosa.com/docs/latest/administration/#version-0-9) for this release.* + ### Added - Add ability to dynamically resize clusters ([#982](https://github.com/pilosa/pilosa/pull/982), [#946](https://github.com/pilosa/pilosa/pull/946), [#929](https://github.com/pilosa/pilosa/pull/929), [#927](https://github.com/pilosa/pilosa/pull/927), [#917](https://github.com/pilosa/pilosa/pull/917), [#913](https://github.com/pilosa/pilosa/pull/913), [#912](https://github.com/pilosa/pilosa/pull/912), [#908](https://github.com/pilosa/pilosa/pull/908)) From 7cecfd30cf7c5354cc926f150c15ef2e9aa7f650 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 4 May 2018 17:07:39 -0500 Subject: [PATCH 79/79] Changelog fixes --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c84b835..5ab4ad1bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ This version contains 188 contribution from 12 contributors. There are 141 files - Add QueryResult.Type to protobuf message to distiguish results at the client ([#1064](https://github.com/pilosa/pilosa/pull/1064)) - Modify `pilosa import` to support string rows/columns ([#1063](https://github.com/pilosa/pilosa/pull/1063)) - Add some statsd calls to HolderSyncer ([#1048](https://github.com/pilosa/pilosa/pull/1048)) -- Adds support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) +- Add support for memberlist gossip configuration via pilosa.Config ([#1014](https://github.com/pilosa/pilosa/pull/1014)) - Add local and cluster IDs ([#1013](https://github.com/pilosa/pilosa/pull/1013), [#1245](https://github.com/pilosa/pilosa/pull/1245)) - Add HolderCleaner and view.DeleteFragment ([#985](https://github.com/pilosa/pilosa/pull/985)) - Add set-coordinator endpoint ([#963](https://github.com/pilosa/pilosa/pull/963)) @@ -45,10 +45,9 @@ This version contains 188 contribution from 12 contributors. There are 141 files - Use binary search in runAdd ([#1027](https://github.com/pilosa/pilosa/pull/1027)) - Use HTTP handler for gossip SendSync ([#1001](https://github.com/pilosa/pilosa/pull/1001)) - Group the write operations in syncBlock by MaxWritesPerRequest ([#950](https://github.com/pilosa/pilosa/pull/950)) -- Refactored HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) +- Refactor HTTPClient handling ([#991](https://github.com/pilosa/pilosa/pull/991)) - Remove FrameSchema. Move Fields to the Frame struct ([#907](https://github.com/pilosa/pilosa/pull/907)) -- Deprecated RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) -- Refactoring pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) +- Refactor pilosa/server ([#1220](https://github.com/pilosa/pilosa/pull/1220)) - Clean up flipBitmap and add tests ([#1223](https://github.com/pilosa/pilosa/pull/1223)) - Move pilosa.Config to pilosa/server.Config ([#1216](https://github.com/pilosa/pilosa/pull/1216)) - Vendor github.com/golang/groupcache/lru ([#1221](https://github.com/pilosa/pilosa/pull/1221)) @@ -80,6 +79,10 @@ This version contains 188 contribution from 12 contributors. There are 141 files - Fix crash due to server.diagnostics.server not set ([#1229](https://github.com/pilosa/pilosa/pull/1229)) - Fix some cluster race conditions ([#1228](https://github.com/pilosa/pilosa/pull/1228)) +### Deprecated + +- Deprecate RangeEnabled option ([#1205](https://github.com/pilosa/pilosa/pull/1205)) + ### Performance - Add benchmark for various container usage patterns ([#1017](https://github.com/pilosa/pilosa/pull/1017))